diff --git a/Cargo.lock b/Cargo.lock index 049b760d9..d94a44573 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1679,6 +1679,7 @@ dependencies = [ "filmcraft-render", "filmcraft-time", "filmcraft-ui-egui", + "glob", "log", "muda", "objc2 0.6.4", @@ -1887,6 +1888,7 @@ dependencies = [ "filmcraft-testkit", "filmcraft-text", "filmcraft-time", + "filmcraft-tts", "image", "log", "serde", @@ -2233,6 +2235,28 @@ dependencies = [ "serde", ] +[[package]] +name = "filmcraft-tts" +version = "0.5.0" +dependencies = [ + "candle-core", + "candle-nn", + "filmcraft-audio-dsp", + "filmcraft-tts-text", + "serde", + "serde_json", + "thiserror 2.0.21", + "zip", +] + +[[package]] +name = "filmcraft-tts-text" +version = "0.5.0" +dependencies = [ + "serde", + "thiserror 2.0.21", +] + [[package]] name = "filmcraft-ui-egui" version = "0.5.0" @@ -2258,6 +2282,7 @@ dependencies = [ "filmcraft-scopes", "filmcraft-text", "filmcraft-time", + "filmcraft-tts", "image", "log", "pollster 0.4.0", @@ -2838,6 +2863,12 @@ dependencies = [ "vello_common", ] +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + [[package]] name = "glow" version = "0.17.0" diff --git a/Cargo.toml b/Cargo.toml index 58a216420..1d306cffa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,6 +44,8 @@ filmcraft-interchange = { path = "crates/interchange" } filmcraft-text = { path = "crates/text" } filmcraft-captions = { path = "crates/captions" } filmcraft-speech = { path = "crates/speech" } +filmcraft-tts = { path = "crates/tts" } +filmcraft-tts-text = { path = "crates/tts-text" } filmcraft-render = { path = "crates/render" } filmcraft-gpu = { path = "crates/gpu" } filmcraft-scopes = { path = "crates/scopes" } diff --git a/apps/filmcraft-cli/src/main.rs b/apps/filmcraft-cli/src/main.rs index e1ba36de2..b051d6cbf 100644 --- a/apps/filmcraft-cli/src/main.rs +++ b/apps/filmcraft-cli/src/main.rs @@ -259,9 +259,15 @@ async fn cli() { register_hardware_decoders(); register_gpu_frame_renderer(); let a = Args::parse(std::env::args().skip(1)); + // `--help` anywhere (`filmcraft-cli --help`, `filmcraft-cli export --help`) prints the reference: + // the parser takes it as a flag option, so it never reaches the subcommand match. + if a.flag("--help") { + out!("{HELP}"); + return; + } let Some(cmd) = a.pos(0) else { usage("missing subcommand") }; match cmd { - "help" | "--help" | "-h" => out!("{HELP}"), + "help" | "-h" => out!("{HELP}"), "version" => outln!("filmcraft-cli {}", env!("CARGO_PKG_VERSION")), "probe" => { let path = a.pos(1).unwrap_or_else(|| usage("probe [--image-sequence]")); diff --git a/apps/filmcraft-cli/tests/cli.rs b/apps/filmcraft-cli/tests/cli.rs index c3d3c979f..2736806e5 100644 --- a/apps/filmcraft-cli/tests/cli.rs +++ b/apps/filmcraft-cli/tests/cli.rs @@ -24,6 +24,22 @@ fn help_and_usage_errors() { assert_eq!(cli(&["exec", "file.newBin", "notkv"]).status.code(), Some(2)); } +#[test] +fn long_help_matches_help_command() { + let reference = cli(&["help"]); + assert!(reference.status.success()); + let long_help = cli(&["--help"]); + assert!(long_help.status.success(), "--help failed: {}", String::from_utf8_lossy(&long_help.stderr)); + assert_eq!(long_help.stdout, reference.stdout); + assert!(long_help.stderr.is_empty()); + let short_help = cli(&["-h"]); + assert!(short_help.status.success()); + assert_eq!(short_help.stdout, reference.stdout); + let sub_help = cli(&["export", "--help"]); + assert!(sub_help.status.success(), "export --help failed: {}", String::from_utf8_lossy(&sub_help.stderr)); + assert_eq!(sub_help.stdout, reference.stdout); +} + #[test] fn probe_reports_mpeg_transport_and_program_streams() { let Some(ffmpeg) = ["/opt/homebrew/bin/ffmpeg", "/usr/local/bin/ffmpeg", "/usr/bin/ffmpeg"].into_iter().find(|p| std::path::Path::new(p).exists()) else { diff --git a/apps/filmcraft/Cargo.toml b/apps/filmcraft/Cargo.toml index 3700e19c5..cf1cfff55 100644 --- a/apps/filmcraft/Cargo.toml +++ b/apps/filmcraft/Cargo.toml @@ -14,6 +14,8 @@ workspace = true default = [] # Speech-to-text (Text panel ▸ Transcribe) with Whisper on the CPU, and model downloads. whisper = ["filmcraft-engine/whisper", "filmcraft-engine/speech-download"] +# Natural Text to Speech voices (Kokoro-82M on the CPU, downloaded on first use). +neural-voices = ["filmcraft-engine/neural-voices"] [dependencies] filmcraft-engine = { workspace = true } @@ -33,6 +35,9 @@ cpal = "0.16" chrono = { version = "0.4", default-features = false, features = ["clock"] } sys-locale = "0.3" +[dev-dependencies] +glob = "0.3" + [target.'cfg(target_os = "macos")'.dependencies] muda = "0.21" objc2-foundation = { version = "0.3", default-features = false, features = ["std", "NSProcessInfo", "NSString", "NSArray"] } diff --git a/apps/filmcraft/src/audio.rs b/apps/filmcraft/src/audio.rs index c8fe8b7f9..f0e9bc489 100644 --- a/apps/filmcraft/src/audio.rs +++ b/apps/filmcraft/src/audio.rs @@ -1,9 +1,10 @@ //! cpal audio output: the playback master clock. Settings ▸ Audio Hardware picks the host //! ("Device Class"), the output device, the I/O buffer size and the sample rate. -use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; + use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use filmcraft_engine::settings::AudioHardwarePrefs; use filmcraft_ui_egui::{AudioDevices, AudioOut}; @@ -17,6 +18,31 @@ pub struct CpalOut { hw: AudioHardwarePrefs, /// Sequence sample rate for "Attempt to force hardware to document sample rate". document_rate: Option, + /// Status-bar note: the host's default device could not be opened and a fallback device is + /// in use, or no output device was found (playing without sound). + note: Option, + /// The fallback picked when the default device could not be opened, as (host, device) names, + /// so later lookups (`devices`, `configure`, `start`) reuse it instead of probing every PCM. + fallback: Mutex>, +} + +/// A hardware note for the status bar, translated when it is shown. +#[derive(Clone, Debug, PartialEq)] +enum Note { + /// The default output device cannot be opened; this named device is used instead. + Fallback(String), + /// No output device at all: playback runs without sound. + NoDevice, +} + +impl Note { + fn text(&self) -> String { + use filmcraft_ui_egui::i18n::{fmt, t}; + match self { + Note::Fallback(device) => fmt(t("Audio hardware: the default output device cannot be opened; using '{device}'"), &[("device", device)]), + Note::NoDevice => t("No audio output device found: playing without sound (check Settings ▸ Audio Hardware)").to_string(), + } + } } /// The host named in the settings (empty or unknown = the default host). @@ -30,21 +56,76 @@ fn host(name: &str) -> cpal::Host { cpal::default_host() } -/// The output device named in the settings (empty or missing = the host's default). -fn output_device(h: &cpal::Host, name: &str) -> Option { +/// The output device the settings ask for, with a status-bar note when a fallback is in use. +struct DeviceChoice { + device: cpal::Device, + /// The host's default device could not be opened: this device is the fallback. + fallback: Option, +} + +/// Fallback order when the default device cannot be opened: the sound-server bridges first +/// (`pipewire`, `pulse`: they share the card with every other app), then other named PCMs, and +/// raw `hw:`/`plughw:` devices last (opening one takes the card exclusively and silences the +/// rest of the desktop). +fn fallback_rank(name: &str) -> u8 { + let n = name.to_ascii_lowercase(); + if n == "pipewire" || n.starts_with("pipewire:") { + 0 + } else if n == "pulse" || n.starts_with("pulse:") { + 1 + } else if n.starts_with("hw:") || n.starts_with("plughw:") { + 3 + } else { + 2 + } +} + +/// The output device named in the settings (empty or missing = the host's default). When the +/// default cannot be opened — a broken ALSA `default` (missing `99-pipewire-default.conf`, +/// #23/#106) — a fallback that can, so the app is not silently mute. The fallback list is probed +/// only after the default fails, in [`fallback_rank`] order, and the pick is cached in `cache`. +fn output_device(h: &cpal::Host, name: &str, cache: &Mutex>) -> Option { if !name.is_empty() && let Ok(mut devs) = h.output_devices() && let Some(d) = devs.find(|d| d.name().is_ok_and(|n| n == name)) { - return Some(d); + return Some(DeviceChoice { device: d, fallback: None }); } - h.default_output_device() + if let Some(d) = h.default_output_device() + && d.default_output_config().is_ok() + { + return Some(DeviceChoice { device: d, fallback: None }); + } + let host_name = h.id().name().to_string(); + let mut cache = cache.lock().unwrap_or_else(PoisonError::into_inner); + let mut devs: Vec<(String, cpal::Device)> = h.output_devices().ok()?.filter_map(|d| d.name().ok().map(|n| (n, d))).collect(); + if let Some((cached_host, cached)) = cache.as_ref() + && *cached_host == host_name + && let Some(i) = devs.iter().position(|(n, _)| n == cached) + { + let (found, d) = devs.swap_remove(i); + return Some(DeviceChoice { device: d, fallback: Some(Note::Fallback(found)) }); + } + devs.sort_by_key(|(n, _)| fallback_rank(n)); + let (found, d) = devs.into_iter().find(|(_, d)| d.default_output_config().is_ok())?; + log::warn!("the default audio output device cannot be opened; falling back to '{found}'"); + *cache = Some((host_name, found.clone())); + Some(DeviceChoice { device: d, fallback: Some(Note::Fallback(found)) }) } impl CpalOut { pub fn new() -> Self { let host = cpal::default_host(); - let cfg = host.default_output_device().and_then(|dev| dev.default_output_config().ok()); + let fallback = Mutex::new(None); + let choice = output_device(&host, "", &fallback); + let cfg = choice.as_ref().and_then(|c| c.device.default_output_config().ok()); + let note = match choice { + Some(c) => c.fallback, + None => { + log::warn!("no audio output device found"); + Some(Note::NoDevice) + } + }; Self { stream: None, played: Arc::new(AtomicU64::new(0)), @@ -53,6 +134,8 @@ impl CpalOut { channels: cfg.as_ref().map_or(2, |c| c.channels()), hw: AudioHardwarePrefs::default(), document_rate: None, + note, + fallback, } } @@ -124,7 +207,8 @@ impl AudioOut for CpalOut { fn start(&mut self, mut fill: Box) -> Result { self.stop(); let host = host(&self.hw.device_class); - let dev = output_device(&host, &self.hw.default_output).ok_or("no output device")?; + let DeviceChoice { device: dev, fallback } = output_device(&host, &self.hw.default_output, &self.fallback).ok_or("no output device")?; + self.note = fallback; let cfg = self.config(&dev)?; let channels = cfg.channels() as usize; if channels == 0 || cfg.sample_rate().0 == 0 { @@ -185,7 +269,8 @@ impl AudioOut for CpalOut { let h = host(&self.hw.device_class); let outputs = h.output_devices().map(|d| d.filter_map(|x| x.name().ok()).collect()).unwrap_or_default(); let inputs = h.input_devices().map(|d| d.filter_map(|x| x.name().ok()).collect()).unwrap_or_default(); - let output_channels = output_device(&h, &self.hw.default_output).and_then(|d| self.config(&d).ok()).map(|c| c.channels()).unwrap_or(0); + let output_channels = + output_device(&h, &self.hw.default_output, &self.fallback).and_then(|c| self.config(&c.device).ok()).map(|cfg| cfg.channels()).unwrap_or(0); AudioDevices { hosts: cpal::available_hosts().iter().map(|h| h.name().to_string()).collect(), inputs, outputs, output_channels } } fn configure(&mut self, hw: &AudioHardwarePrefs, document_rate: Option) { @@ -193,13 +278,17 @@ impl AudioOut for CpalOut { self.document_rate = document_rate; // the rate playback mixes at must be known before `start` let host = host(&self.hw.device_class); - if let Some(dev) = output_device(&host, &self.hw.default_output) - && let Ok(cfg) = self.config(&dev) + if let Some(choice) = output_device(&host, &self.hw.default_output, &self.fallback) + && let Ok(cfg) = self.config(&choice.device) { self.rate = cfg.sample_rate().0; self.channels = cfg.channels(); + self.note = choice.fallback; } } + fn note(&self) -> Option { + self.note.as_ref().map(Note::text) + } } #[cfg(test)] @@ -207,6 +296,19 @@ mod tests { use super::*; use cpal::Sample; + #[test] + fn fallback_prefers_sound_servers_over_raw_hardware() { + let mut names = vec!["plughw:CARD=PCH,DEV=0", "hw:CARD=PCH,DEV=0", "sysdefault:CARD=PCH", "pulse", "pipewire"]; + names.sort_by_key(|n| fallback_rank(n)); + assert_eq!(names, ["pipewire", "pulse", "sysdefault:CARD=PCH", "plughw:CARD=PCH,DEV=0", "hw:CARD=PCH,DEV=0"]); + } + + #[test] + fn notes_name_the_fallback_device() { + assert!(Note::Fallback("pipewire".into()).text().contains("'pipewire'")); + assert!(!Note::NoDevice.text().is_empty()); + } + #[test] fn integer_conversion_clips_and_preserves_interleaved_channels() { let mut signed = [0_i16; 8]; diff --git a/apps/filmcraft/src/dev_icon.rs b/apps/filmcraft/src/dev_icon.rs new file mode 100644 index 000000000..8c9876794 --- /dev/null +++ b/apps/filmcraft/src/dev_icon.rs @@ -0,0 +1,329 @@ +//! Dev-time Linux desktop entry: the shell icon fallback. +//! +//! The Linux shell (Wayland *and* X11) resolves the window to a `.desktop` file by app id (set in +//! `main.rs` via `eframe`'s `with_app_id`) and takes the taskbar / window-list icon from its +//! `Icon=` entry in the hicolor theme. Packaging installs both +//! (`packaging/linux/ai.storyteller.filmcraft.desktop` and the hicolor PNGs) — but a source-tree +//! run has neither, so KDE/GNOME fall back to the generic Wayland icon. The fix is to ensure a +//! user-level entry for those runs: when no entry for the app id exists in any XDG data dir, +//! install one (plus the hicolor icon PNGs, read from this checkout's `assets/app-icon`, so +//! nothing is embedded in the binary) into `$XDG_DATA_HOME`/`~/.local/share`. An existing +//! packaged entry is never overridden; the user-owned dev copy is rewritten when it points at +//! another binary (a dev build moves between checkouts). The shell picks the icon up at the *next* app start; +//! `update-desktop-database`/`kbuildsycoca` is not needed for icon lookup. +//! +//! Opt-in only: nothing is written unless `FILMCRAFT_DEV_DESKTOP_ENTRY=1` is set, so ordinary +//! `cargo run`s (agents, worktrees, CI) never touch the user's profile. Packaged runs are +//! excluded even then — AppImage (`APPIMAGE`) mounts at a fresh `/tmp/.mount_*` each launch, +//! so its `Exec` path changes every time and it ships its own entry, and a Flatpak +//! (`FLATPAK_ID`) installs its entry through the manifest. `packaging/linux/install.sh` writes the packaged +//! `Exec=` the same two-layer way as here; keep the two in sync. +//! +//! Every failure is logged and skipped — a missing dev icon must never stop the app +//! ([`AGENTS.md`](/AGENTS.md) §0 "never crash"). + +/// Install the user-level desktop entry and icons for a source-tree run (Linux). +pub fn ensure_dev_desktop_entry() { + // Packaged and sandboxed runs never write into the user's profile: an AppImage mounts at a + // fresh /tmp/.mount_* every launch (the Exec path changes each time) and a Flatpak installs + // its own entry. + if std::env::var_os("APPIMAGE").is_some() || std::env::var_os("FLATPAK_ID").is_some() { + return; + } + // Opt-in only: a plain `cargo run` (agents, worktrees, other checkouts) must not write into + // the user's profile, and a packaged release never does (packages and install.sh install + // the real entry). + if std::env::var_os("FILMCRAFT_DEV_DESKTOP_ENTRY").is_none_or(|v| v != "1") { + return; + } + if let Err(e) = ensure() { + log::warn!("desktop entry (taskbar icon) not installed: {e}"); + } +} + +/// Install missing files; `Ok` whether anything was written or an entry already existed. +fn ensure() -> Result<(), DevIconError> { + let file_name = format!("{}.desktop", crate::APP_ID); + let user_dir = user_data_dir()?; + let exe = std::env::current_exe().map_err(DevIconError::Io)?; + // Not `to_string_lossy`: a substituted U+FFFD would install an `Exec` that can never start + // the app, and the same lossy string would match on every later run, so the broken entry + // would never be rewritten or warned about. + let exe = exe.to_str().ok_or(DevIconError::BadExecPath)?; + if exe.contains('=') || exe.contains('%') || exe.chars().any(|c| c.is_control()) { + // A desktop entry cannot carry these (install.sh rejects them for the same reason): + // '=' separates key and value, '%' introduces a field code, control characters are + // not valid in the string layer. + return Err(DevIconError::BadExecPath); + } + for dir in search_dirs(&user_dir) { + let candidate = dir.join("applications").join(&file_name); + if !candidate.is_file() { + continue; + } + // A packaged entry (any system dir) already gives the shell its icon; only our own + // user-level copy is kept fresh. + if dir != user_dir { + return Ok(()); + } + let body = std::fs::read_to_string(&candidate).map_err(DevIconError::Io)?; + if desktop_file_exec_is_ours(&body, exe) { + return Ok(()); + } + break; // a dev entry from another checkout: fall through and rewrite it + } + for size in ICON_SIZES { + let rel = std::path::Path::new("hicolor").join(format!("{size}x{size}")).join("apps").join(format!("{}.png", crate::APP_ID)); + let dest = user_dir.join("icons").join(&rel); + if !dest.is_file() + && let Some(parent) = dest.parent() + { + // The checkout this binary was built from; a missing icon only costs the icon. + let src = std::path::Path::new(ICON_DIR).join(&rel); + if !src.is_file() { + log::warn!("dev desktop entry: icon {} not found; skipping it", src.display()); + continue; + } + std::fs::create_dir_all(parent).map_err(DevIconError::Io)?; + std::fs::copy(&src, &dest).map_err(DevIconError::Io)?; + } + } + let applications = user_dir.join("applications"); + std::fs::create_dir_all(&applications).map_err(DevIconError::Io)?; + let body = desktop_file_content(crate::APP_ID, "FilmCraft", exe); + std::fs::write(applications.join(&file_name), body).map_err(DevIconError::Io)?; + Ok(()) +} + +/// The hicolor icon sizes installed for the taskbar (a subset large enough for panel and topbar; +/// the shell picks the nearest size, so 256 and 512 cover both). +const ICON_SIZES: [u16; 2] = [256, 512]; + +/// The app icons in the source tree this binary was built from (a path, not embedded bytes: +/// the entry is a dev-checkout convenience and release binaries must not carry the PNGs). +const ICON_DIR: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/../../assets/app-icon"); + +/// The user data dir we install into: `$XDG_DATA_HOME`, else `$HOME/.local/share` (XDG base +/// directory spec). Untrusted environment: an empty or relative `XDG_DATA_HOME` means the +/// default, and a missing or relative `HOME` is an error — never a guess that would write +/// into the working directory. +fn user_data_dir() -> Result { + if let Some(v) = std::env::var_os("XDG_DATA_HOME") + && !v.is_empty() + && std::path::Path::new(&v).is_absolute() + { + return Ok(std::path::PathBuf::from(v)); + } + match std::env::var_os("HOME").filter(|h| !h.is_empty() && std::path::Path::new(h).is_absolute()) { + Some(home) => Ok(std::path::PathBuf::from(home).join(".local").join("share")), + None => Err(DevIconError::NoHome), + } +} + +/// The XDG data dirs to search for an existing entry, user dir first, then `$XDG_DATA_DIRS` (or +/// the spec default `/usr/local/share:/usr/share`). Only absolute paths are taken; the spec +/// requires them and a relative one would read the working directory. +fn search_dirs(user: &std::path::Path) -> Vec { + let mut dirs = vec![user.to_path_buf()]; + let system = std::env::var_os("XDG_DATA_DIRS") + .filter(|v| !v.is_empty()) + .map(|v| v.to_string_lossy().into_owned()) + .unwrap_or_else(|| "/usr/local/share:/usr/share".into()); + for dir in system.split(':') { + if std::path::Path::new(dir).is_absolute() { + dirs.push(std::path::PathBuf::from(dir)); + } + } + dirs +} + +/// Whether the `.desktop` body's `Exec` points at our binary exactly (a stale dev entry from +/// another checkout — `/a/filmcraft-old` is not `/a/filmcraft` — does not, and is rewritten; a +/// packaged `Exec=filmcraft` never matches a source-tree path). +fn desktop_file_exec_is_ours(body: &str, exe: &str) -> bool { + exec_line(body).is_some_and(|arg| arg == exe) +} + +/// The first `Exec=` argument (the executable), unescaped through both Desktop Entry layers: +/// first the string layer of the file value (`\\` `\s` `\n` `\t` `\r`, per +/// ), then the shell layer of the +/// argument (double quotes, and `\` before `` ` `"` `\` `$` inside them). `None` when there is no +/// `Exec=` line or its first argument cannot be parsed. +fn exec_line(body: &str) -> Option { + let value = body.lines().find(|l| l.starts_with("Exec=")).and_then(|l| l.strip_prefix("Exec="))?; + let value = unescape_string_value(value); + if let Some(rest) = value.strip_prefix('"') { + let mut out = String::new(); + let mut chars = rest.chars(); + while let Some(c) = chars.next() { + match c { + '"' => return Some(out), + '\\' => match chars.next() { + Some(e @ ('\\' | '"' | '`' | '$')) => out.push(e), + Some(e) => { + out.push('\\'); + out.push(e); + } + None => return None, // a dangling escape ends the argument + }, + c => out.push(c), + } + } + None // no closing quote + } else { + Some(value.split(' ').next().unwrap_or_default().to_string()) + } +} + +/// Undo the desktop entry's string layer: `\\` `\s` `\n` `\t` `\r` are the defined sequences; a +/// backslash before anything else is kept as the character itself (lenient, like desktop file +/// parsers in the wild). +fn unescape_string_value(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + let mut chars = value.chars(); + while let Some(c) = chars.next() { + if c != '\\' { + out.push(c); + continue; + } + match chars.next() { + Some('\\') => out.push('\\'), + Some('s') => out.push(' '), + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some('r') => out.push('\r'), + Some(e) => out.push(e), + None => out.push('\\'), + } + } + out +} + +/// The `.desktop` body: same fields as `packaging/linux/ai.storyteller.filmcraft.desktop`, with +/// `Exec` pointing at this exact binary (a dev build isn't on `$PATH`, so `TryExec` is left out) +/// and the icon by theme name. +fn desktop_file_content(app_id: &str, name: &str, exe: &str) -> String { + let mut out = String::new(); + out.push_str("[Desktop Entry]\n"); + out.push_str("Type=Application\n"); + out.push_str(&format!("Name={name}\n")); + out.push_str("GenericName=Video Editor\n"); + out.push_str("Comment=Edit video, color and sound\n"); + out.push_str(&format!("Exec=\"{}\" %F\n", escape_exec_argument(exe))); + out.push_str(&format!("Icon={app_id}\n")); + out.push_str("Terminal=false\n"); + out.push_str("StartupNotify=true\n"); + out.push_str(&format!("StartupWMClass={app_id}\n")); + out.push_str("Categories=AudioVideo;Video;AudioVideoEditing;\n"); + out.push_str("Keywords=video;editor;film;timeline;color;grading;nle;\n"); + out +} + +/// Escape `s` for the `Exec=` value's two layers, like `packaging/linux/install.sh`: first the +/// shell layer (`` ` `` `"` `$` `\` get a backslash, and the argument is double-quoted), then +/// the string layer of the file value (every backslash doubled). +fn escape_exec_argument(s: &str) -> String { + let mut shell = String::with_capacity(s.len() + 2); + for c in s.chars() { + if matches!(c, '`' | '"' | '$' | '\\') { + shell.push('\\'); + } + shell.push(c); + } + shell.replace('\\', "\\\\") +} + +/// Everything that can keep the dev icon out of the taskbar; logged, never fatal. +#[derive(Debug)] +enum DevIconError { + Io(std::io::Error), + NoHome, + BadExecPath, +} + +impl std::fmt::Display for DevIconError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + DevIconError::Io(e) => write!(f, "{e}"), + DevIconError::NoHome => write!(f, "no home directory for the user data dir"), + DevIconError::BadExecPath => { + write!(f, "the binary path has a character a desktop entry cannot carry (=, % or a control character), or is not valid UTF-8") + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The entry mirrors the packaged one, derived from the real file so the two cannot drift: + /// minus `TryExec` (a dev build isn't on `$PATH`), plus the quoted dev `Exec` path. + /// The icons the entry installs exist in the checkout. + #[test] + fn icon_sources_exist_in_the_checkout() { + for size in ICON_SIZES { + let p = std::path::Path::new(ICON_DIR).join(format!("hicolor/{size}x{size}/apps/ai.storyteller.filmcraft.png")); + assert!(p.is_file(), "{}", p.display()); + } + } + + #[test] + fn desktop_file_matches_the_packaged_fields() { + let packaged = include_str!("../../../packaging/linux/ai.storyteller.filmcraft.desktop"); + let expected = packaged + .lines() + .filter(|l| !l.starts_with("TryExec=")) + .map(|l| if l.starts_with("Exec=") { "Exec=\"/target/debug/filmcraft\" %F" } else { l }) + .collect::>() + .join("\n") + + "\n"; + let body = desktop_file_content("ai.storyteller.filmcraft", "FilmCraft", "/target/debug/filmcraft"); + assert_eq!(body, expected); + } + + /// Both layers are escaped, like install.sh: shell first (`` ` `` `"` `$` `\` get a + /// backslash), then *every* backslash doubled for the file value — including the escape + /// backslashes just added. + #[test] + fn exec_escapes_both_desktop_entry_layers() { + assert_eq!(escape_exec_argument("/opt/film craft/filmcraft"), "/opt/film craft/filmcraft"); + // a\b → shell a\\b → string a\\\\b + assert_eq!(escape_exec_argument("a\\b"), "a\\\\\\\\b"); + // a$b → shell a\$b → string a\\$b + assert_eq!(escape_exec_argument("a$b"), "a\\\\$b"); + } + + /// A written entry reads back as the exact binary it was written for, through both layers. + #[test] + fn exec_round_trips_through_both_layers() { + for exe in ["/home/dev/Projects/filmcraft/target/debug/filmcraft", "/opt/film craft/filmcraft", "/a$b\"c`d\\e"] { + let body = desktop_file_content("ai.storyteller.filmcraft", "FilmCraft", exe); + assert_eq!(exec_line(&body).as_deref(), Some(exe)); + assert!(desktop_file_exec_is_ours(&body, exe)); + } + } + + /// Only the exact executable counts: another checkout's path, a path with our path as a + /// prefix, and a packaged `Exec=filmcraft` are all someone else's entry. + #[test] + fn desktop_file_exec_is_ours_matches_only_our_binary() { + let body = desktop_file_content("ai.storyteller.filmcraft", "FilmCraft", "/a/filmcraft"); + assert!(!desktop_file_exec_is_ours(&body, "/a/filmcraft-old")); + assert!(!desktop_file_exec_is_ours(&body, "/b/filmcraft")); + assert!(!desktop_file_exec_is_ours("Exec=filmcraft %F\n", "/a/filmcraft")); + assert!(!desktop_file_exec_is_ours("Name=filmcraft\n", "/a/filmcraft")); + assert!(!desktop_file_exec_is_ours("Exec=\"dangling\n", "/a/filmcraft")); + } + + /// The string layer is undone before the argument is split: `\s` becomes a space, which + /// ends a bare argument (the shell layer's quoting, not `\s`, is how a space travels), and + /// `\\` is one backslash. install.sh always quotes, so a space in our own `Exec` never + /// splits. + #[test] + fn exec_reads_the_string_layer() { + assert_eq!(exec_line("Exec=/a\\sb %F\n").as_deref(), Some("/a")); + assert_eq!(exec_line("Exec=\"/a\\sb\" %F\n").as_deref(), Some("/a b")); + assert_eq!(exec_line("Exec=/a\\\\b %F\n").as_deref(), Some("/a\\b")); + } +} diff --git a/apps/filmcraft/src/file_filters.rs b/apps/filmcraft/src/file_filters.rs new file mode 100644 index 000000000..bc7dab594 --- /dev/null +++ b/apps/filmcraft/src/file_filters.rs @@ -0,0 +1,70 @@ +//! Extension filters for native file dialogs. +//! +//! Every filtered file dialog (import, relink, open file/preset, open/save project) builds +//! its filter with [`extensions`]. On Linux and the BSDs, rfd's XDG portal, GTK and Zenity +//! backends turn each extension into a case-sensitive `*.{ext}` glob, so camera files such +//! as `shot.MP4` or `shot.Mp4` would be hidden. The helper appends one bracket pattern per +//! extension (`mp4` -> `[mM][pP]4`) after the literal extensions, which stay first so save +//! dialogs keep their normal default suffix. macOS and Windows dialogs already ignore case +//! and receive the plain list. + +pub fn extensions(exts: &[&str]) -> Vec { + let extensions: Vec<_> = exts.iter().map(|ext| (*ext).to_owned()).collect(); + // XDG portal/Zenity globs are case-sensitive. Keep the literal extensions first so + // Save dialogs can still infer a normal default suffix, then cover mixed case too. + // https://flatpak.github.io/xdg-desktop-portal/docs/doc-org.freedesktop.portal.FileChooser.html + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + { + let mut extensions = extensions; + for ext in exts { + if !ext.bytes().any(|b| b.is_ascii_alphabetic()) { + continue; + } + let mut pattern = String::new(); + for c in ext.chars() { + if c.is_ascii_alphabetic() { + pattern.extend(['[', c.to_ascii_lowercase(), c.to_ascii_uppercase(), ']']); + } else { + pattern.push(c); + } + } + extensions.push(pattern); + } + extensions + } + // Native macOS and Windows filters take ordinary extensions. + #[cfg(any(target_os = "windows", target_os = "macos"))] + extensions +} + +#[cfg(test)] +mod tests { + use super::extensions; + + // rfd's XDG portal and Zenity backends turn each extension into `*.{ext}`. + // Exercise those case-sensitive patterns against filenames, as the picker does. + #[cfg(not(any(target_os = "windows", target_os = "macos")))] + #[test] + fn camera_files_are_visible_in_every_capitalization() { + let filters: Vec<_> = extensions(&["mp4", "mov", "m2ts", "wav", "fcproj"]).iter().map(|ext| glob::Pattern::new(&format!("*.{ext}")).unwrap()).collect(); + for name in ["camera.mp4", "camera.MP4", "camera.Mp4", "clip.MOV", "clip.moV", "take.M2TS", "sound.WaV", "Edit.FCPROJ", "Edit.FcProj"] { + assert!(filters.iter().any(|f| f.matches(name)), "the picker hides {name}"); + } + for name in ["camera.mp4.bak", "camera.mp44", "camera.xmp4", "notes.txt", "mp4"] { + assert!(!filters.iter().any(|f| f.matches(name)), "the picker admits {name}"); + } + } + + #[test] + fn save_filters_keep_the_literal_default_extension() { + assert_eq!(extensions(&["mp4", "mov"]).first().map(String::as_str), Some("mp4")); + assert!(extensions(&[]).is_empty()); + assert_eq!(extensions(&["*", "", "123"]), ["*", "", "123"]); + } + + #[cfg(any(target_os = "windows", target_os = "macos"))] + #[test] + fn native_filters_receive_only_literal_extensions() { + assert_eq!(extensions(&["mp4", "mov", "fcproj"]), ["mp4", "mov", "fcproj"]); + } +} diff --git a/apps/filmcraft/src/main.rs b/apps/filmcraft/src/main.rs index b58a5084c..8e9668d6d 100644 --- a/apps/filmcraft/src/main.rs +++ b/apps/filmcraft/src/main.rs @@ -28,6 +28,9 @@ mod args; mod audio; mod audio_in; mod control_server; +#[cfg(target_os = "linux")] +mod dev_icon; +mod file_filters; #[cfg(any(target_os = "windows", test))] mod graphics; mod logging; @@ -107,6 +110,8 @@ fn main() -> eframe::Result { } } app_nap::disable(); + #[cfg(target_os = "linux")] + dev_icon::ensure_dev_desktop_entry(); // Panics anywhere go to /Logs/crash-.log with a backtrace; the UI pass and // frame workers catch them and keep running (see filmcraft_ui_egui::crash). filmcraft_ui_egui::crash::install(log_dir); @@ -200,7 +205,7 @@ fn main() -> eframe::Result { app.audio = Some(Box::new(audio::CpalOut::new())); app.hooks.pick_files = Some(Box::new(|exts: &[&str]| { rfd::FileDialog::new() - .add_filter(filmcraft_ui_egui::i18n::t("Media"), exts) + .add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts)) .pick_files() .unwrap_or_default() .into_iter() @@ -209,18 +214,21 @@ fn main() -> eframe::Result { })); // Link Media ▸ Locate…, Attach Proxies, Reconnect Full Resolution: one path, not imported. app.hooks.pick_file_for_relink = Some(Box::new(|exts: &[&str], _hint| { - rfd::FileDialog::new().add_filter(filmcraft_ui_egui::i18n::t("Media"), exts).pick_file().map(|p| p.to_string_lossy().to_string()) + rfd::FileDialog::new() + .add_filter(filmcraft_ui_egui::i18n::t("Media"), &file_filters::extensions(exts)) + .pick_file() + .map(|p| p.to_string_lossy().to_string()) })); app.hooks.pick_save = Some(Box::new(|name: &str| { rfd::FileDialog::new() - .add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &["fcproj"]) + .add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"])) .set_file_name(name) .save_file() .map(|p| p.to_string_lossy().to_string()) })); app.hooks.pick_save_as = Some(Box::new(|filter: &str, exts: &[&str], name: &str| { rfd::FileDialog::new() - .add_filter(filmcraft_ui_egui::i18n::t(filter), exts) + .add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts)) .set_file_name(name) .save_file() .map(|p| p.to_string_lossy().to_string()) @@ -230,6 +238,7 @@ fn main() -> eframe::Result { Some(Box::new(|dir: &str| rfd::FileDialog::new().set_directory(dir).pick_folder().map(|p| p.to_string_lossy().into_owned()))); app.hooks.open_path = Some(Box::new(open_path)); // Settings ▸ General ▸ Interface Language ▸ System Language (#218). + app.hooks.cursor_screen_position = Some(Box::new(filmcraft_platform::cursor::cursor_screen_position)); app.hooks.system_languages = Some(Box::new(|| sys_locale::get_locales().collect())); // Settings ▸ Appearance ▸ Appearance Mode ▸ Sync with system on Linux desktops whose // compositor reports no theme to winit (no polling: see appearance.rs). @@ -238,11 +247,14 @@ fn main() -> eframe::Result { window_raise::raise_without_focus(); })); app.hooks.pick_open_file = Some(Box::new(|filter: &str, exts: &[&str]| { - rfd::FileDialog::new().add_filter(filmcraft_ui_egui::i18n::t(filter), exts).pick_file().map(|p| p.to_string_lossy().to_string()) + rfd::FileDialog::new() + .add_filter(filmcraft_ui_egui::i18n::t(filter), &file_filters::extensions(exts)) + .pick_file() + .map(|p| p.to_string_lossy().to_string()) })); app.hooks.pick_open_project = Some(Box::new(|| { rfd::FileDialog::new() - .add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &["fcproj"]) + .add_filter(filmcraft_ui_egui::i18n::t("FilmCraft Project"), &file_filters::extensions(&["fcproj"])) .pick_file() .map(|p| p.to_string_lossy().to_string()) })); diff --git a/contributors/people.toml b/contributors/people.toml index 9762e2ac7..50788355b 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -22,6 +22,11 @@ real_name = "Brandon Thomas" display_name = "Brandon Thomas" sync_github_name = false +[people.ahmedhosnypro] +real_name = "Ahmed Hosny" +display_name = "Ahmed Hosny" +sync_github_name = false + [people.sanyamsingh001] display_name = "sanyamsingh001" sync_github_name = false @@ -41,6 +46,10 @@ sync_github_name = true [people.ItzBloxy] display_name = "Bloxy" +[people.Hathor875] +real_name = "Krzysztof Cieślik" +sync_github_name = true + [people.ReReddo] display_name = "ReReddo" sync_github_name = false diff --git a/crates/automation/src/server.rs b/crates/automation/src/server.rs index 4c705c3c2..0aebbc31d 100644 --- a/crates/automation/src/server.rs +++ b/crates/automation/src/server.rs @@ -139,6 +139,30 @@ pub struct TypeParams { pub text: String, } +#[derive(Debug, Deserialize, JsonSchema)] +pub struct MediaImportParams { + /// Absolute paths to media files (MP4/MOV, WAV/MP3/FLAC/AIFF/Ogg, PNG/JPEG/…). + pub paths: Vec, + /// Bin ID to import into (optional). + #[serde(default)] + pub bin: Option, + /// Import numbered stills as an image sequence (optional, default false). + #[serde(default)] + pub image_sequence: Option, +} + +/// `file.import` params for a `media_import` call. +fn media_import_params(p: MediaImportParams) -> Value { + let mut params = json!({"paths": p.paths}); + if let Some(bin) = p.bin { + params["bin"] = json!(bin); + } + if let Some(seq) = p.image_sequence { + params["imageSequence"] = json!(seq); + } + params +} + #[derive(Debug, Default, Deserialize, JsonSchema)] pub struct ElementsParams { /// Only element ids starting with this prefix (e.g. `timeline.clip.`). @@ -381,9 +405,8 @@ impl FilmcraftMcp { description = "Import media files by absolute path (MP4/MOV, WAV/MP3/FLAC/AIFF/Ogg, PNG/JPEG/…).", annotations(read_only_hint = false, destructive_hint = false, idempotent_hint = false, open_world_hint = false) )] - async fn media_import(&self, Parameters(p): Parameters) -> Result { - let paths: Vec<&str> = p.text.split('\n').map(str::trim).filter(|s| !s.is_empty()).collect(); - wrap(self.run("file.import", json!({"paths": paths})).await) + async fn media_import(&self, Parameters(p): Parameters) -> Result { + wrap(self.run("file.import", media_import_params(p)).await) } #[tool( @@ -735,6 +758,21 @@ mod tests { } } + /// #499: `media_import` takes `paths` (a required array), and forwards `bin` / `image_sequence` to `file.import`. + #[test] + fn media_import_schema_and_params() { + let m = FilmcraftMcp::headless(Session::default()); + let def = m.tool_router.get("media_import").unwrap(); + let schema = Value::Object((*def.input_schema).clone()); + assert_eq!(schema["properties"]["paths"]["type"], "array", "{schema}"); + assert!(schema["required"].as_array().unwrap().iter().any(|r| r == "paths"), "{schema}"); + let p: MediaImportParams = serde_json::from_value(json!({"paths": ["/a.png"], "bin": 7, "image_sequence": true})).unwrap(); + assert_eq!(media_import_params(p), json!({"paths": ["/a.png"], "bin": 7, "imageSequence": true})); + let p: MediaImportParams = serde_json::from_value(json!({"paths": ["/a.wav"]})).unwrap(); + assert_eq!(media_import_params(p), json!({"paths": ["/a.wav"]})); + assert!(serde_json::from_value::(json!({"text": "/a.wav"})).is_err()); + } + fn demo() -> Session { let mut s = Session::default(); s.execute("file.openDemoProject", json!({})).unwrap(); diff --git a/crates/edit/src/lib.rs b/crates/edit/src/lib.rs index dc228522a..2f5a8382b 100644 --- a/crates/edit/src/lib.rs +++ b/crates/edit/src/lib.rs @@ -22,6 +22,7 @@ pub mod frame_size; pub mod multicam; pub mod through; pub mod transcript; +pub mod transitions; use std::collections::HashMap; diff --git a/crates/edit/src/transitions.rs b/crates/edit/src/transitions.rs new file mode 100644 index 000000000..c487c7d24 --- /dev/null +++ b/crates/edit/src/transitions.rs @@ -0,0 +1,401 @@ +//! Applied transitions: where one may sit over its cut, its alignment, and moving / resizing / +//! removing one (Effect Controls' Duration and Alignment, dragging its edges or its middle in the +//! Timeline, Delete). +//! +//! Premiere (Align transitions; Change transition duration): a transition between two clips stays +//! over their cut, "Center at Cut", "Start at Cut", "End at Cut" or, once dragged off-centre, +//! "Custom Start". Changing its duration moves both ends equally for Center at Cut / Custom Start, +//! only the end for Start at Cut and only the beginning for End at Cut. A one-sided transition (a +//! fade from or to nothing) starts or ends at its clip's edge. +//! +//! Premiere limits a transition by the media its clips have beyond the cut (handles). FilmCraft +//! does not check handles when applying one, so here a transition may cover at most the clips it +//! joins: from the outgoing clip's start to the incoming clip's end. + +use filmcraft_project::{Sequence, Track, Transition, TransitionAlign, TransitionId}; +use filmcraft_time::Tick; + +use crate::{Edge, EditError, Result}; + +/// How a transition sits on its cut, as Effect Controls' Alignment menu shows it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Alignment { + Center, + Start, + End, + /// Premiere's "Custom Start": dragged to a place over the cut that is none of the others. + Custom, +} + +impl Alignment { + pub const MENU: [Alignment; 3] = [Alignment::Center, Alignment::Start, Alignment::End]; + + pub fn id(self) -> &'static str { + match self { + Alignment::Center => "center", + Alignment::Start => "start", + Alignment::End => "end", + Alignment::Custom => "custom", + } + } + + pub fn label(self) -> &'static str { + match self { + Alignment::Center => "Center at Cut", + Alignment::Start => "Start at Cut", + Alignment::End => "End at Cut", + Alignment::Custom => "Custom Start", + } + } + + /// `center`, `start` or `end` (Custom Start is reached by moving a transition, not chosen). + pub fn parse(s: &str) -> Option { + Alignment::MENU.into_iter().find(|a| a.id().eq_ignore_ascii_case(s) || a.label().eq_ignore_ascii_case(s)) + } +} + +/// Where a transition may be: `(earliest start, cut, latest end)`. Between two clips that is the +/// outgoing clip's start, the cut and the incoming clip's end; for a fade to or from nothing the cut +/// is its clip's edge. `None` when its clips are gone. +pub fn bounds(track: &Track, tr: &Transition) -> Option<(Tick, Tick, Tick)> { + let from = tr.from.and_then(|c| track.item(c)); + let to = tr.to.and_then(|c| track.item(c)); + match (from, to) { + (Some(f), Some(t)) => Some((f.start, t.start, t.end())), + (None, Some(t)) => Some((t.start, t.start, t.end())), + (Some(f), None) => Some((f.start, f.end(), f.end())), + (None, None) => None, + } +} + +/// A transition's alignment on `cut`. Centred means within half a `frame` of the cut, as +/// centring an odd number of frames lands on a frame boundary. +pub fn alignment(tr: &Transition, cut: Tick, frame: Tick) -> Alignment { + match (tr.from, tr.to) { + (None, Some(_)) => return Alignment::Start, + (Some(_), None) => return Alignment::End, + _ => {} + } + if tr.start == cut { + Alignment::Start + } else if tr.end() == cut { + Alignment::End + } else if (tr.start.0.saturating_mul(2).saturating_add(tr.duration.0) - cut.0.saturating_mul(2)).abs() <= frame.0.max(0) { + Alignment::Center + } else { + Alignment::Custom + } +} + +fn find(seq: &Sequence, id: TransitionId) -> Option<(&Track, &Transition)> { + seq.all_tracks().find_map(|t| t.transitions.iter().find(|x| x.id == id).map(|x| (t, x))) +} + +/// Where a transition would start for `duration` and `align`, keeping its cut (and, for Center +/// at Cut / Custom Start, its centre; `snap` puts a centred start on a frame). +pub fn start_for(seq: &Sequence, id: TransitionId, duration: Tick, align: Alignment, frame: Tick, snap: impl Fn(Tick) -> Tick) -> Option { + let (track, tr) = find(seq, id)?; + let (_, cut, _) = bounds(track, tr)?; + let one_sided = alignment(tr, cut, frame); + let align = match one_sided { + Alignment::Start | Alignment::End if tr.from.is_none() || tr.to.is_none() => one_sided, + _ => align, + }; + Some(match align { + Alignment::Start => cut, + Alignment::End => cut - duration, + Alignment::Center => snap(cut - duration.mul_ratio(1, 2)), + // Custom Start: the ends move equally, around where it is now + Alignment::Custom => snap(tr.start + tr.duration.mul_ratio(1, 2) - duration.mul_ratio(1, 2)), + }) +} + +/// Move / resize a transition to cover `start .. start + duration`. Refused (leaving the sequence +/// as it was) when its track is locked, it would be shorter than `min`, it would leave its clips, +/// or it would no longer touch its cut (a one-sided one must keep its edge). +pub fn set_span(seq: &mut Sequence, id: TransitionId, start: Tick, duration: Tick, min: Tick, frame: Tick) -> Result<()> { + let (track, tr) = find(seq, id).ok_or_else(|| EditError::Other(format!("no transition {}", id.0)))?; + if track.locked { + return Err(EditError::Locked); + } + let (lo, cut, hi) = bounds(track, tr).ok_or_else(|| EditError::Other("the transition's clips are gone".into()))?; + if duration < min.max(Tick(1)) { + return Err(EditError::Other("a transition lasts at least one frame".into())); + } + let end = start.0.checked_add(duration.0).map(Tick).ok_or_else(|| EditError::Other("transition out of range".into()))?; + if start < lo || end > hi { + return Err(EditError::Other("a transition can't extend past the clips it joins".into())); + } + let on_cut = match (tr.from, tr.to) { + (None, Some(_)) => start == cut, + (Some(_), None) => end == cut, + _ => start <= cut && cut <= end, + }; + if !on_cut { + return Err(EditError::Other(if tr.from.is_some() && tr.to.is_some() { + "a transition between two clips must stay over their cut".into() + } else { + "a one-sided transition stays on its clip's edge".into() + })); + } + let tid = track.id; + let Some(x) = seq.track_mut(tid).and_then(|t| t.transitions.iter_mut().find(|x| x.id == id)) else { + return Err(EditError::Other(format!("no transition {}", id.0))); + }; + x.start = start; + x.duration = duration; + // the stored alignment (interchange): Custom Start has no variant, its span says it all + x.align = match alignment(x, cut, frame) { + Alignment::Start => TransitionAlign::StartAtCut, + Alignment::End => TransitionAlign::EndAtCut, + Alignment::Center | Alignment::Custom => TransitionAlign::CenterAtCut, + }; + Ok(()) +} + +/// Where dragging a transition by `delta` in the Timeline puts it, as `(start, duration)`. Its +/// middle (`edge` None) slides it over the cut. An end trims that end only (Premiere shows the +/// Trim-In / Trim-Out icon there): the other end stays, whatever the alignment, so a Center at +/// Cut transition becomes Custom Start. (A typed duration, in Effect Controls or Set Transition +/// Duration, follows the alignment instead: see [`start_for`].) A fade to or from nothing keeps +/// its clip-edge end, and does not slide. Always clamped to where it may be ([`bounds`], over the +/// cut, at least one `frame`). `None` when its clips are gone. +pub fn drag_span(track: &Track, tr: &Transition, edge: Option, delta: Tick, frame: Tick) -> Option<(Tick, Tick)> { + let (lo, cut, hi) = bounds(track, tr)?; + let frame = frame.max(Tick(1)); + let (s0, e0, d0) = (tr.start, tr.end(), tr.duration); + let one_sided = tr.from.is_none() || tr.to.is_none(); + let unchanged = Some((s0, d0)); + match edge { + None if one_sided => unchanged, + None => { + let (min_s, max_s) = (lo.max(cut - d0), (hi - d0).min(cut)); + if min_s > max_s { unchanged } else { Some(((s0 + delta).clamp(min_s, max_s), d0)) } + } + Some(Edge::In) if tr.from.is_none() => unchanged, + Some(Edge::In) => { + let max_s = cut.min(e0 - frame).max(lo); + let st = (s0 + delta).clamp(lo, max_s); + Some((st, e0 - st)) + } + Some(Edge::Out) if tr.to.is_none() => unchanged, + Some(Edge::Out) => { + let min_e = cut.max(s0 + frame).min(hi); + let en = (e0 + delta).clamp(min_e, hi); + Some((s0, en - s0)) + } + } +} + +/// Remove transitions (on unlocked tracks). Returns how many went. +pub fn remove(seq: &mut Sequence, ids: &[TransitionId]) -> Result { + if seq.all_tracks().any(|t| t.locked && t.transitions.iter().any(|x| ids.contains(&x.id))) { + return Err(EditError::Locked); + } + let mut n = 0; + for t in seq.all_tracks_mut() { + let before = t.transitions.len(); + t.transitions.retain(|x| !ids.contains(&x.id)); + n += before - t.transitions.len(); + } + if n == 0 { Err(EditError::Nothing) } else { Ok(n) } +} + +#[cfg(test)] +mod tests { + use super::*; + use filmcraft_project::{ClipId, ItemId, Label, Project, SequenceSettings, TrackItem, find_effect}; + use filmcraft_time::FrameRate; + + const R: FrameRate = FrameRate::FPS_24; + + fn f(n: i64) -> Tick { + R.tick_of(n) + } + + fn clip(id: u64, start: i64, dur: i64) -> TrackItem { + TrackItem { + id: ClipId(id), + item: ItemId(1), + name: format!("c{id}"), + label: Label::Iris, + start: f(start), + duration: f(dur), + source_in: Tick::ZERO, + speed: 1.0, + reverse: false, + enabled: true, + link: None, + group: None, + effects: vec![], + markers: vec![], + gain_db: 0.0, + frame_hold: None, + scale_to_frame: false, + essential: None, + multicam: None, + time_interpolation: Default::default(), + hold_filters: false, + field_options: None, + source_channels: Vec::new(), + audio_stream: 0, + graphic: None, + } + } + + fn transition(id: u64, start: i64, dur: i64, from: Option, to: Option) -> Transition { + // any effect: these tests are about where the transition sits + let effect = find_effect("opacity").unwrap().instance(); + Transition { + id: TransitionId(id), + effect, + start: f(start), + duration: f(dur), + from: from.map(ClipId), + to: to.map(ClipId), + align: TransitionAlign::CenterAtCut, + reverse: false, + } + } + + /// V1: c1 0..48, c2 48..96, c3 96..144; a centred 24-frame transition on 48 (id 1), a fade in + /// at 0 (id 2) and a fade out at 144 (id 3). + fn seq() -> Sequence { + let mut p = Project::new("t"); + let s = p.new_sequence("s", SequenceSettings { frame_rate: R, ..Default::default() }, 1, 0, None); + let mut q = p.sequence(s).unwrap().clone(); + let t = &mut q.video_tracks[0]; + t.items = vec![clip(1, 0, 48), clip(2, 48, 48), clip(3, 96, 48)]; + t.transitions = vec![transition(1, 36, 24, Some(1), Some(2)), transition(2, 0, 12, None, Some(1)), transition(3, 132, 12, Some(3), None)]; + q + } + + fn span(q: &Sequence, id: u64) -> (Tick, Tick) { + let x = q.video_tracks[0].transitions.iter().find(|x| x.id.0 == id).unwrap(); + (x.start, x.duration) + } + + fn align(q: &Sequence, id: u64) -> Alignment { + let (t, x) = find(q, TransitionId(id)).unwrap(); + let (_, cut, _) = bounds(t, x).unwrap(); + alignment(x, cut, f(1)) + } + + #[test] + fn alignment_reads_the_span() { + let mut q = seq(); + assert_eq!((align(&q, 1), align(&q, 2), align(&q, 3)), (Alignment::Center, Alignment::Start, Alignment::End)); + set_span(&mut q, TransitionId(1), f(48), f(24), f(1), f(1)).unwrap(); + assert_eq!(align(&q, 1), Alignment::Start); + set_span(&mut q, TransitionId(1), f(24), f(24), f(1), f(1)).unwrap(); + assert_eq!(align(&q, 1), Alignment::End); + set_span(&mut q, TransitionId(1), f(40), f(24), f(1), f(1)).unwrap(); + assert_eq!(align(&q, 1), Alignment::Custom); + // an odd centred duration sits half a frame off the cut and is still Center at Cut + set_span(&mut q, TransitionId(1), f(42), f(13), f(1), f(1)).unwrap(); + assert_eq!(align(&q, 1), Alignment::Center); + } + + #[test] + fn duration_changes_follow_the_alignment() { + let q = seq(); + let snap = |t: Tick| R.snap(t); + // Center at Cut: both ends move + assert_eq!(start_for(&q, TransitionId(1), f(36), Alignment::Center, f(1), snap), Some(f(30))); + // Start at Cut: only the end moves; End at Cut: only the beginning + assert_eq!(start_for(&q, TransitionId(1), f(36), Alignment::Start, f(1), snap), Some(f(48))); + assert_eq!(start_for(&q, TransitionId(1), f(36), Alignment::End, f(1), snap), Some(f(12))); + // a one-sided transition keeps its edge whatever is asked + assert_eq!(start_for(&q, TransitionId(2), f(20), Alignment::Center, f(1), snap), Some(f(0))); + assert_eq!(start_for(&q, TransitionId(3), f(20), Alignment::Center, f(1), snap), Some(f(124))); + } + + #[test] + fn custom_start_resizes_around_its_middle() { + let mut q = seq(); + set_span(&mut q, TransitionId(1), f(40), f(20), f(1), f(1)).unwrap(); + assert_eq!(start_for(&q, TransitionId(1), f(10), Alignment::Custom, f(1), |t| R.snap(t)), Some(f(45))); + } + + #[test] + fn spans_stay_on_the_cut_and_inside_the_clips() { + let mut q = seq(); + let before = q.clone(); + for (id, start, dur) in [ + (1, -1, 24), // before the outgoing clip + (1, 40, 60), // past the incoming clip's end + (1, 10, 20), // entirely before the cut + (1, 50, 10), // entirely after it + (1, 40, 0), // empty + (2, 1, 12), // a fade in must start at its clip's start + (3, 120, 12), // a fade out must end at its clip's end + (9, 0, 12), // no such transition + ] { + assert!(set_span(&mut q, TransitionId(id), f(start), f(dur), f(1), f(1)).is_err(), "{id} {start} {dur}"); + assert_eq!(q, before, "refused changes leave the sequence as it was"); + } + // the whole of both clips is allowed + set_span(&mut q, TransitionId(1), f(0), f(96), f(1), f(1)).unwrap(); + assert_eq!(span(&q, 1), (f(0), f(96))); + // Tick::MAX never overflows + assert!(set_span(&mut q, TransitionId(1), Tick(i64::MAX - 1), Tick(i64::MAX), f(1), f(1)).is_err()); + } + + fn drag(q: &Sequence, id: u64, edge: Option, frames: i64) -> (i64, i64) { + let (t, x) = find(q, TransitionId(id)).unwrap(); + let (st, d) = drag_span(t, x, edge, f(frames), f(1)).unwrap(); + (R.frame_at(st), R.frame_at(d)) + } + + #[test] + fn dragging_a_transition() { + let mut q = seq(); + // centred (36..60 over the cut at 48): an end trims that end only, the other stays + assert_eq!(drag(&q, 1, Some(Edge::In), -4), (32, 28)); + assert_eq!(drag(&q, 1, Some(Edge::Out), -10), (36, 14)); + assert_eq!(drag(&q, 1, Some(Edge::Out), -100), (36, 12), "the end stops at the cut"); + assert_eq!(drag(&q, 1, Some(Edge::Out), 100), (36, 60), "and at the incoming clip's end"); + assert_eq!(drag(&q, 1, Some(Edge::In), -100), (0, 60), "the start at the outgoing clip's start"); + // the middle slides it, staying over the cut and inside the clips + assert_eq!(drag(&q, 1, None, 5), (41, 24)); + assert_eq!(drag(&q, 1, None, 100), (48, 24)); + assert_eq!(drag(&q, 1, None, -100), (24, 24)); + // not centred: the other end stays + set_span(&mut q, TransitionId(1), f(40), f(20), f(1), f(1)).unwrap(); + assert_eq!(drag(&q, 1, Some(Edge::In), -4), (36, 24)); + assert_eq!(drag(&q, 1, Some(Edge::In), 100), (48, 12), "the start stops at the cut"); + assert_eq!(drag(&q, 1, Some(Edge::Out), 6), (40, 26)); + assert_eq!(drag(&q, 1, Some(Edge::Out), -100), (40, 8), "the end stops at the cut"); + // a fade in keeps its start, a fade out its end; neither slides + assert_eq!(drag(&q, 2, Some(Edge::In), 3), (0, 12)); + assert_eq!(drag(&q, 2, Some(Edge::Out), 3), (0, 15)); + assert_eq!(drag(&q, 2, None, 3), (0, 12)); + assert_eq!(drag(&q, 3, Some(Edge::Out), 3), (132, 12)); + assert_eq!(drag(&q, 3, Some(Edge::In), -3), (129, 15)); + // whatever comes out is a span set_span accepts + for (id, edge, n) in [(1, Some(Edge::In), -70), (1, None, 33), (2, Some(Edge::Out), 99), (3, Some(Edge::In), -99)] { + let (t, x) = find(&q, TransitionId(id)).unwrap(); + let (st, d) = drag_span(t, x, edge, f(n), f(1)).unwrap(); + let mut c = q.clone(); + set_span(&mut c, TransitionId(id), st, d, f(1), f(1)).unwrap(); + } + } + + #[test] + fn locked_tracks_refuse() { + let mut q = seq(); + q.video_tracks[0].locked = true; + assert_eq!(set_span(&mut q, TransitionId(1), f(40), f(20), f(1), f(1)), Err(EditError::Locked)); + assert_eq!(remove(&mut q, &[TransitionId(1)]), Err(EditError::Locked)); + } + + #[test] + fn remove_takes_only_the_named_transitions() { + let mut q = seq(); + assert_eq!(remove(&mut q, &[TransitionId(1), TransitionId(3)]), Ok(2)); + let left: Vec = q.video_tracks[0].transitions.iter().map(|x| x.id.0).collect(); + assert_eq!(left, vec![2]); + assert_eq!(q.video_tracks[0].items.len(), 3, "the clips stay"); + assert_eq!(remove(&mut q, &[TransitionId(1)]), Err(EditError::Nothing)); + } +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 5c8065935..defe08a87 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -15,6 +15,8 @@ default = [] whisper = ["filmcraft-speech/whisper"] # Speech-model downloads over HTTPS. speech-download = ["filmcraft-speech/download"] +# Natural voices for Text to Speech: Kokoro-82M on candle (CPU) and their download. +neural-voices = ["filmcraft-tts/kokoro", "speech-download"] [dependencies] filmcraft-time = { workspace = true } @@ -35,6 +37,7 @@ filmcraft-format = { workspace = true } filmcraft-interchange = { workspace = true } filmcraft-captions = { workspace = true } filmcraft-speech = { workspace = true } +filmcraft-tts = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } diff --git a/crates/engine/src/clip_ops.rs b/crates/engine/src/clip_ops.rs index f5343f7fd..ec4804a3d 100644 --- a/crates/engine/src/clip_ops.rs +++ b/crates/engine/src/clip_ops.rs @@ -949,6 +949,7 @@ fn consolidate_duplicates(s: &mut Session, _: &Value) -> Result { for it in pr.items.values_mut() { match &mut it.kind { ItemKind::Sequence(q) => { + let q = std::sync::Arc::make_mut(q); for t in q.all_tracks_mut() { for i in &mut t.items { if let Some(k) = map.get(&i.item) { diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index 28dd24a19..0b3a9f8a6 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -81,6 +81,15 @@ pub(crate) fn has_selection(s: &Session) -> std::result::Result<(), String> { } /// Clips or captions selected (Clear / Ripple Delete work on either). fn has_any_selection(s: &Session) -> std::result::Result<(), String> { + has_seq(s)?; + if s.state.selection.is_empty() && s.state.caption_selection.is_empty() && s.state.transition_selection.is_empty() { + Err("nothing selected".into()) + } else { + Ok(()) + } +} +/// Clips or captions selected (Ripple Delete: a transition has no length of its own to close). +fn has_clip_or_caption_selection(s: &Session) -> std::result::Result<(), String> { has_seq(s)?; if s.state.selection.is_empty() && s.state.caption_selection.is_empty() { Err("nothing selected".into()) } else { Ok(()) } } @@ -1204,6 +1213,11 @@ fn build() -> Vec { cmd!("edit.pasteInsert", "Paste Insert", ["Edit"], Some("Cmd+Shift+V"), "{}", has_clipboard, |s, _| paste(s, true)), cmd!("edit.clear", "Clear", ["Edit"], Some("Backspace"), r#"{"clips":[id]?}"#, has_any_selection, |s, p| { if p.get("clips").is_none() && p.get("clip").is_none() && s.state.selection.is_empty() { + // a selected transition (clicked in the Timeline) goes, as a clip would + if !s.state.transition_selection.is_empty() { + let ids = s.state.transition_selection.clone(); + return remove_transitions(s, &ids, "Clear"); + } let caps = s.state.caption_selection.clone(); return crate::captions::delete(s, &caps, false); } @@ -1215,7 +1229,7 @@ fn build() -> Vec { })?; Ok(Value::Null) }), - cmd!("edit.rippleDelete", "Ripple Delete", ["Edit"], Some("Shift+Delete"), r#"{"clips":[id]?}"#, has_any_selection, |s, p| { + cmd!("edit.rippleDelete", "Ripple Delete", ["Edit"], Some("Shift+Delete"), r#"{"clips":[id]?}"#, has_clip_or_caption_selection, |s, p| { if p.get("clips").is_none() && p.get("clip").is_none() && s.state.selection.is_empty() { let caps = s.state.caption_selection.clone(); return crate::captions::delete(s, &caps, true); @@ -1363,10 +1377,70 @@ fn build() -> Vec { let sel = clips_p(s, p); let linked = sel.iter().all(|c| s.active_sequence().and_then(|q| q.find_item(*c)).is_some_and(|(_, i)| i.link.is_some())); s.edit_sequence(if linked { "Unlink" } else { "Link" }, |q, ctx, _| { - let l = if linked { None } else { Some(ctx.alloc()) }; - for c in &sel { - if let Some((_, i)) = q.find_item_mut(*c) { - i.link = l; + if linked { + for c in &sel { + if let Some((_, i)) = q.find_item_mut(*c) { + i.link = None; + } + } + } else { + // each selected clip with its track kind, worked out once + type Spec = (ClipId, filmcraft_project::ItemId, filmcraft_time::TimeRange); + let (mut v_clips, mut audio_specs): (Vec, Vec) = (Vec::new(), Vec::new()); + for c in &sel { + if let Some((tid, i)) = q.find_item(*c) { + if q.video_tracks.iter().any(|t| t.id == tid) { + v_clips.push((*c, i.item, i.range())); + } else { + audio_specs.push((*c, i.item, i.range())); + } + } + } + if !audio_specs.is_empty() && v_clips.len() > 1 { + let mut linked_audios = std::collections::HashSet::new(); + for (vid, item_id, vrange) in &v_clips { + // Pair with overlapping audio clips from the same item, or overlapping audio clips if no same-item audio + let matching: Vec = audio_specs + .iter() + .filter(|(aid, aitem, arange)| !linked_audios.contains(aid) && aitem == item_id && arange.overlaps(vrange)) + .map(|(aid, _, _)| *aid) + .collect(); + let to_link = if !matching.is_empty() { + matching + } else { + audio_specs + .iter() + .filter(|(aid, _, arange)| !linked_audios.contains(aid) && arange.overlaps(vrange)) + .map(|(aid, _, _)| *aid) + .collect() + }; + // a video clip with no audio partner stays unlinked: a one-member link would read as "linked" + let l = (!to_link.is_empty()).then(|| ctx.alloc()); + if let Some((_, vi)) = q.find_item_mut(*vid) { + vi.link = l; + } + for aid in to_link { + linked_audios.insert(aid); + if let Some((_, ai)) = q.find_item_mut(aid) { + ai.link = l; + } + } + } + // selected audio clips with no overlapping video stay unlinked + for (aid, _, _) in audio_specs { + if !linked_audios.contains(&aid) + && let Some((_, ai)) = q.find_item_mut(aid) + { + ai.link = None; + } + } + } else { + let l = ctx.alloc(); + for c in &sel { + if let Some((_, i)) = q.find_item_mut(*c) { + i.link = Some(l); + } + } } } Ok(()) @@ -1557,10 +1631,20 @@ fn build() -> Vec { "Edit Transition Settings", [], None, - r#"{"transition":id,"params":{param:value}?,"reverse":bool?,"reset":bool?}"#, + r#"{"transition":id,"params":{param:value}?,"reverse":bool?,"reset":bool?,"duration":ticks?|"frames":n?,"align":"center"|"start"|"end"?|"start":ticks?,"merge":bool?,"begin":bool?}"#, has_seq, set_transition ), + cmd!("sequence.removeTransition", "Clear Transition", [], None, r#"{"transitions":[id]?}"#, has_seq, |s, p| { + let ids: Vec = match p.get("transitions").or_else(|| p.get("transition")) { + Some(v) => transition_ids_p(v).ok_or_else(|| bad("sequence.removeTransition", "`transitions` is a list of transition ids"))?, + None => s.state.transition_selection.clone(), + }; + if ids.is_empty() { + return Err(bad("sequence.removeTransition", "no transition given or selected")); + } + remove_transitions(s, &ids, "Clear Transition") + }), cmd!("sequence.closeGap", "Close Gap", ["Sequence"], None, r#"{"track":"V1"|id,"time":ticks}"#, has_seq, |s, p| { let tr = track_p(s, p, "track", "sequence.closeGap")?.ok_or_else(|| bad("sequence.closeGap", "need `track`"))?; let t = time_p(s, p, "").unwrap_or(s.playhead()); @@ -2052,13 +2136,18 @@ fn build() -> Vec { Ok(json!({"clips": ids.iter().map(|c| c.0).collect::>()})) } ), - cmd!("timeline.select", "Select Clips", [], None, r#"{"clips":[id],"add":bool,"toggle":bool}"#, has_seq, |s, p| { + cmd!("timeline.select", "Select Clips", [], None, r#"{"clips":[id],"transitions":[id]?,"add":bool,"toggle":bool}"#, has_seq, |s, p| { + if let Some(ids) = p.get("transitions") { + return select_transitions(s, ids, bool_p(p, "add").unwrap_or(false), bool_p(p, "toggle").unwrap_or(false)); + } let clips: Vec = p.get("clips").and_then(Value::as_array).map(|a| a.iter().filter_map(|v| v.as_u64().map(ClipId)).collect()).unwrap_or_default(); let clips = with_links(s, &clips); // selecting clips leaves trim mode (Premiere: clip and edit point selections are exclusive) s.state.edit_points.clear(); s.state.trim_shift = Default::default(); + // and a clip selection replaces a transition one + s.state.transition_selection.clear(); if bool_p(p, "toggle").unwrap_or(false) { for c in clips { if let Some(i) = s.state.selection.iter().position(|x| *x == c) { @@ -2432,13 +2521,14 @@ fn build() -> Vec { "Set Effect Parameter", [], None, - r##"{"clip":id,"effect":"motion"|index,"param":str,"mask":n?,"value":num|[x,y]|"#rrggbb"|bool|path,"time":ticks?,"merge":bool?,"begin":bool?}"##, + r##"{"clip":id,"effect":"motion"|index,"param":str,"mask":n?,"value":num|[x,y]|"#rrggbb"|bool|path|"option name" (a choice takes its index or its name),"time":ticks? (where the keyframe goes when the parameter is animated; a parameter that is not animated gets a new static value whatever the time),"keyframe":bool? (true: write a keyframe at `time` and animate the parameter),"merge":bool?,"begin":bool?}"##, has_seq, |s, p| { let c = clip_p(p, "clip").ok_or_else(|| bad("effects.setParam", "need `clip`"))?; let pid = str_p(p, "param").ok_or_else(|| bad("effects.setParam", "need `param`"))?.to_string(); let val = p.get("value").cloned().ok_or_else(|| bad("effects.setParam", "need `value`"))?; let eff = p.get("effect").cloned().unwrap_or(json!("motion")); + let keyframe = bool_p(p, "keyframe").unwrap_or(false); let ph = s.playhead(); let tl = time_p(s, p, "").unwrap_or(ph); let pq = p.clone(); @@ -2447,7 +2537,7 @@ fn build() -> Vec { if bool_p(p, "begin").unwrap_or(false) { s.history.merge_key = None; } - s.edit_sequence_as("Change Effect Parameter", merge.as_deref(), |q, _, _| { + let keyframes = s.edit_sequence_as("Change Effect Parameter", merge.as_deref(), |q, _, _| { let (_, it) = q.find_item_mut(c).ok_or(filmcraft_edit::EditError::NoItem(c))?; let mt = it.source_time_at(tl.clamp(it.start, (it.end() - Tick(1)).max(it.start))); let e = match &eff { @@ -2456,6 +2546,22 @@ fn build() -> Vec { _ => None, } .ok_or_else(|| bad("effects.setParam", "no such effect on clip"))?; + // the parameter's definition (a mask's own parameters have none) + let def = if pq.get("mask").is_none_or(Value::is_null) { e.def().and_then(|d| d.param(&pid)) } else { None }; + // a choice takes its option name as well as its index, as transition parameters do + let val = match (def.map(|d| &d.kind), &val) { + (Some(filmcraft_project::ParamKind::Choice(opts)), Value::String(name)) => { + let i = opts + .iter() + .position(|o| o.eq_ignore_ascii_case(name)) + .ok_or_else(|| bad("effects.setParam", format!("`{pid}` must be one of {opts:?}")))?; + json!(i) + } + _ => val.clone(), + }; + if keyframe && def.is_some_and(|d| !d.animatable) { + return Err(bad("effects.setParam", format!("`{pid}` cannot be animated"))); + } if eff == json!("opacity") || eff == json!("motion") { e.enabled = true; } @@ -2467,10 +2573,16 @@ fn build() -> Vec { } let prm = crate::masks::target_param(e, &pq, &pid).ok_or_else(|| bad("effects.setParam", format!("no param `{pid}`")))?; let v = json_to_param(&prm.value, &val).ok_or_else(|| bad("effects.setParam", "value has the wrong type"))?; - prm.set_at(mt, v); - Ok(()) + if keyframe { + // the stopwatch and the value in one step + prm.put_keyframe(mt, v); + } else { + prm.set_at(mt, v); + } + Ok(prm.keyframes.len()) })?; - Ok(Value::Null) + // 0 keyframes: the value is static (`time` was not used) + Ok(json!({"keyframes": keyframes})) } ), cmd!("effects.toggleAnimation", "Toggle Animation", [], None, r#"{"clip":id,"effect":str|index,"param":str,"mask":n?}"#, has_seq, |s, p| { @@ -2718,6 +2830,7 @@ fn build() -> Vec { v.extend(crate::scopes::commands()); v.extend(crate::remix::commands()); v.extend(crate::voiceover::commands()); + v.extend(crate::narration::commands()); v.extend(crate::keyboard::commands()); v.extend(crate::project_panel::commands()); v.extend(crate::media_browser::commands()); @@ -3200,13 +3313,109 @@ fn set_transition_params(e: &mut filmcraft_project::EffectInstance, params: &Val Ok(()) } -/// `sequence.setTransition`: edit an applied transition's settings (Effect Controls). +/// A JSON list of transition ids (or one id). +fn transition_ids_p(v: &Value) -> Option> { + match v { + Value::Array(a) => a.iter().map(|x| x.as_u64().map(TransitionId)).collect(), + x => x.as_u64().map(|i| vec![TransitionId(i)]), + } +} + +/// `timeline.select {"transitions": […]}`: select transitions (clicking one in the Timeline). It +/// replaces the clip, edit point and caption selections, as a clip selection replaces it. +fn select_transitions(s: &mut Session, ids: &Value, add: bool, toggle: bool) -> Result { + const CMD: &str = "timeline.select"; + let ids = transition_ids_p(ids).ok_or_else(|| bad(CMD, "`transitions` is a list of transition ids"))?; + let q = s.active_sequence().ok_or(EngineError::NoSequence)?; + if let Some(missing) = ids.iter().find(|id| !q.all_tracks().any(|t| t.transitions.iter().any(|x| x.id == **id))) { + return Err(bad(CMD, format!("no transition {}", missing.0))); + } + if toggle { + for id in ids { + match s.state.transition_selection.iter().position(|x| *x == id) { + Some(i) => { + s.state.transition_selection.remove(i); + } + None => s.state.transition_selection.push(id), + } + } + } else if add { + for id in ids { + if !s.state.transition_selection.contains(&id) { + s.state.transition_selection.push(id); + } + } + } else { + s.state.transition_selection = ids; + } + s.state.selection.clear(); + s.state.caption_selection.clear(); + s.state.edit_points.clear(); + s.state.trim_shift = Default::default(); + Ok(json!({"selection": [], "transitionSelection": s.state.transition_selection.iter().map(|t| t.0).collect::>()})) +} + +/// Remove transitions in one undo step and drop them from the selection. +fn remove_transitions(s: &mut Session, ids: &[TransitionId], label: &str) -> Result { + let ids = ids.to_vec(); + let n = s.edit_sequence(label, |q, _, st| { + let n = edit::transitions::remove(q, &ids)?; + st.transition_selection.retain(|x| !ids.contains(x)); + Ok(n) + })?; + Ok(json!({"removed": n})) +} + +/// `sequence.setTransition`: edit an applied transition (Effect Controls, the Timeline): its +/// settings, Reverse, and where it sits. `duration` / `frames` resize it the way its alignment +/// says (Center at Cut and Custom Start: both ends; Start at Cut: the end; End at Cut: the +/// beginning), `align` re-aligns it on its cut, `start` moves it (dragging it over the cut; with +/// `duration`, dragging one edge). One undo step; with `merge`, a drag's steps share one. fn set_transition(s: &mut Session, p: &Value) -> Result { const CMD: &str = "sequence.setTransition"; let id = p.get("transition").and_then(Value::as_u64).ok_or_else(|| bad(CMD, "need `transition`"))?; + let tid = TransitionId(id); let q = s.active_sequence().ok_or(EngineError::NoSequence)?; - let found = q.video_tracks.iter().chain(q.audio_tracks.iter()).find_map(|tr| tr.transitions.iter().find(|x| x.id.0 == id)).cloned(); - let mut cur = found.ok_or_else(|| bad(CMD, format!("no transition {id}")))?; + let rate = q.settings.frame_rate.sane(); + let frame = rate.frame_duration(); + let found = q.all_tracks().find_map(|tr| tr.transitions.iter().find(|x| x.id == tid).map(|x| (tr, x))); + let (track, cur) = found.ok_or_else(|| bad(CMD, format!("no transition {id}")))?; + let mut cur = cur.clone(); + let (lo, cut, hi) = edit::transitions::bounds(track, &cur).ok_or_else(|| bad(CMD, "the transition's clips are gone"))?; + // times outside the clips it joins are refused before any frame arithmetic (hostile input) + let room = Tick(hi.0.saturating_sub(lo.0)); + let duration = match (p.get("frames").and_then(Value::as_i64), p.get("duration").and_then(Value::as_i64)) { + (Some(_), Some(_)) => return Err(bad(CMD, "give `duration` or `frames`, not both")), + (Some(f), None) if (1..=10_000_000).contains(&f) && rate.tick_of(f) <= room => Some(rate.tick_of(f)), + (None, Some(d)) if d > 0 && d <= room.0 => Some(rate.snap_nearest(Tick(d)).max(frame)), + (Some(_), None) | (None, Some(_)) => return Err(bad(CMD, "a transition lasts at least one frame and can't extend past the clips it joins")), + (None, None) => None, + }; + let align = match str_p(p, "align") { + Some(a) => Some(edit::transitions::Alignment::parse(a).ok_or_else(|| bad(CMD, format!("`align` is center, start or end, not `{a}`")))?), + None => None, + }; + let start = match p.get("start").and_then(Value::as_i64) { + Some(t) if (lo.0..=hi.0).contains(&t) => Some(rate.snap_nearest(Tick(t))), + Some(_) => return Err(bad(CMD, "a transition can't extend past the clips it joins")), + None => None, + }; + if start.is_some() && align.is_some() { + return Err(bad(CMD, "give `start` or `align`, not both")); + } + let span = if duration.is_some() || align.is_some() || start.is_some() { + let d = duration.unwrap_or(cur.duration); + let st = match start { + Some(st) => st, + None => { + let al = align.unwrap_or_else(|| edit::transitions::alignment(&cur, cut, frame)); + edit::transitions::start_for(q, tid, d, al, frame, |t| rate.snap(t)).ok_or_else(|| bad(CMD, format!("no transition {id}")))? + } + }; + Some((st, d)) + } else { + None + }; if p.get("reset").and_then(Value::as_bool).unwrap_or(false) && let Some(def) = cur.effect.def() { @@ -3219,16 +3428,32 @@ fn set_transition(s: &mut Session, p: &Value) -> Result { cur.reverse = r; } let name = cur.effect.def().map_or("Transition", |d| d.name).to_string(); - s.edit_sequence(&format!("Edit {name}"), |q, _, _| { + if bool_p(p, "begin").unwrap_or(false) { + s.history.merge_key = None; + } + let merge = bool_p(p, "merge").unwrap_or(false).then(|| format!("setTransition:{id}")); + let after = s.edit_sequence_as(&format!("Edit {name}"), merge.as_deref(), |q, _, _| { + if let Some((st, d)) = span { + edit::transitions::set_span(q, tid, st, d, frame, frame)?; + } + let mut after = None; for tr in q.video_tracks.iter_mut().chain(q.audio_tracks.iter_mut()) { - if let Some(x) = tr.transitions.iter_mut().find(|x| x.id.0 == id) { + if let Some(x) = tr.transitions.iter_mut().find(|x| x.id == tid) { x.effect = cur.effect.clone(); x.reverse = cur.reverse; + after = Some((x.start, x.duration)); } } - Ok(()) + Ok(after) })?; - Ok(json!({"transition": id, "effect": cur.effect.effect, "reverse": cur.reverse})) + let (st, d) = after.unwrap_or((cur.start, cur.duration)); + let al = s + .active_sequence() + .and_then(|q| { + q.all_tracks().find_map(|tr| tr.transitions.iter().find(|x| x.id == tid).and_then(|x| edit::transitions::bounds(tr, x).map(|b| (x.clone(), b.1)))) + }) + .map(|(x, cut)| edit::transitions::alignment(&x, cut, frame).id()); + Ok(json!({"transition": id, "effect": cur.effect.effect, "reverse": cur.reverse, "start": st.0, "duration": d.0, "align": al})) } fn apply_transition(s: &mut Session, p: &Value, kind: TrackKind) -> Result { @@ -3345,7 +3570,7 @@ pub fn inspect_sequence(s: &Session, id: ItemId, q: &filmcraft_project::Sequence "enabled": i.enabled, "link": i.link, "label": i.label.name(), "effects": i.effects.iter().map(|e| json!({"effect": e.effect, "enabled": e.enabled, "masks": e.masks.len(), "params": e.params.iter().map(|(k, p)| (k.clone(), json!({"value": format!("{:?}", p.value), "keyframes": p.keyframes.len()}))).collect::>()})).collect::>(), })).collect::>(), - "transitions": t.transitions.iter().map(|x| json!({"id": x.id.0, "effect": x.effect.effect, "start": x.start.0, "duration": x.duration.0, "from": x.from.map(|c| c.0), "to": x.to.map(|c| c.0), "reverse": x.reverse, "params": x.effect.params.iter().map(|(k, v)| (k.clone(), serde_json::to_value(&v.value).unwrap_or_default())).collect::>()})).collect::>(), + "transitions": t.transitions.iter().map(|x| json!({"id": x.id.0, "effect": x.effect.effect, "start": x.start.0, "duration": x.duration.0, "from": x.from.map(|c| c.0), "to": x.to.map(|c| c.0), "reverse": x.reverse, "align": filmcraft_edit::transitions::bounds(t, x).map(|b| filmcraft_edit::transitions::alignment(x, b.1, rate.frame_duration()).id()), "params": x.effect.params.iter().map(|(k, v)| (k.clone(), serde_json::to_value(&v.value).unwrap_or_default())).collect::>()})).collect::>(), }) }; json!({ @@ -3362,6 +3587,7 @@ pub fn inspect_sequence(s: &Session, id: ItemId, q: &filmcraft_project::Sequence "video": q.video_tracks.iter().map(tr).collect::>(), "audio": q.audio_tracks.iter().map(tr).collect::>(), "selection": s.state.selection.iter().map(|c| c.0).collect::>(), + "transitionSelection": s.state.transition_selection.iter().map(|t| t.0).collect::>(), }) } diff --git a/crates/engine/src/explicit_targets_tests.rs b/crates/engine/src/explicit_targets_tests.rs index 1f2eca77f..ce377f64d 100644 --- a/crates/engine/src/explicit_targets_tests.rs +++ b/crates/engine/src/explicit_targets_tests.rs @@ -211,3 +211,56 @@ fn project_delete_takes_explicit_bins_and_items_without_a_selection() { s.execute("edit.undo", json!({})).unwrap(); assert_eq!(s.project.items.len(), before); } + +#[test] +fn link_multiple_video_and_audio_pairs_creates_one_link_per_pair() { + let mut s = demo(); + let v_clips = v1(&s); + let a_clips = a1(&s); + assert!(v_clips.len() >= 3 && a_clips.len() >= 3); + let (v0, v1, v2) = (v_clips[0].id, v_clips[1].id, v_clips[2].id); + let (a0, a1, a2) = (a_clips[0].id, a_clips[1].id, a_clips[2].id); + + // First, unlink all three pairs + let r = s.execute("clip.link", json!({"clips": [v0.0, a0.0, v1.0, a1.0, v2.0, a2.0]})).unwrap(); + assert_eq!(r["linked"], false); + for cid in [v0, a0, v1, a1, v2, a2] { + assert_eq!(clip(&s, cid).unwrap().link, None); + } + + // Now, run clip.link on all six clips. In Premiere parity (#507), + // it should create 3 separate links (one per pair), not one big link for all six. + let r = s.execute("clip.link", json!({"clips": [v0.0, a0.0, v1.0, a1.0, v2.0, a2.0]})).unwrap(); + assert_eq!(r["linked"], true); + + let l0 = clip(&s, v0).unwrap().link; + let la0 = clip(&s, a0).unwrap().link; + let l1 = clip(&s, v1).unwrap().link; + let la1 = clip(&s, a1).unwrap().link; + let l2 = clip(&s, v2).unwrap().link; + let la2 = clip(&s, a2).unwrap().link; + + assert!(l0.is_some() && l1.is_some() && l2.is_some()); + assert_eq!(l0, la0, "v0 linked to a0"); + assert_eq!(l1, la1, "v1 linked to a1"); + assert_eq!(l2, la2, "v2 linked to a2"); + + assert_ne!(l0, l1, "different pairs must not share the same link id"); + assert_ne!(l1, l2, "different pairs must not share the same link id"); + assert_ne!(l0, l2, "different pairs must not share the same link id"); +} + +/// A video clip with no overlapping selected audio, and audio with no overlapping video, stay +/// unlinked: a one-member link would make the clip read as "linked" with no partner. +#[test] +fn link_leaves_clips_without_a_partner_unlinked() { + let mut s = demo(); + let (v, a) = (v1(&s), a1(&s)); + let (v0, v1, a0, a2) = (v[0].id, v[1].id, a[0].id, a[2].id); + s.execute("clip.link", json!({"clips": [v0.0, v1.0, a0.0, a2.0]})).unwrap(); + s.execute("clip.link", json!({"clips": [v0.0, v1.0, a0.0, a2.0]})).unwrap(); + assert!(clip(&s, v0).unwrap().link.is_some()); + assert_eq!(clip(&s, v0).unwrap().link, clip(&s, a0).unwrap().link); + assert_eq!(clip(&s, v1).unwrap().link, None, "v1 has no overlapping audio in the selection"); + assert_eq!(clip(&s, a2).unwrap().link, None, "a2 has no overlapping video in the selection"); +} diff --git a/crates/engine/src/graphic_templates.rs b/crates/engine/src/graphic_templates.rs index 0b9941e29..f5053756b 100644 --- a/crates/engine/src/graphic_templates.rs +++ b/crates/engine/src/graphic_templates.rs @@ -291,7 +291,7 @@ fn set_controls(s: &mut Session, clip: ClipId, values: &serde_json::Map Result; @@ -262,10 +262,19 @@ fn apply_props(e: &mut EffectInstance, props: &serde_json::Map, m } /// Set properties `props` on a layer (keyframe-aware at time `tl`). Changing the text keeps -/// per-character styles on their characters. -pub(crate) fn set_props(s: &mut Session, clip: ClipId, eidx: usize, props: &serde_json::Map, tl: Tick, label: &str) -> Result<()> { +/// per-character styles on their characters. `merge` folds consecutive calls with the same key +/// into one undo step (a drag is one undoable change, like `effects.setParam`). +pub(crate) fn set_props( + s: &mut Session, + clip: ClipId, + eidx: usize, + props: &serde_json::Map, + tl: Tick, + label: &str, + merge: Option<&str>, +) -> Result<()> { let props = props.clone(); - s.edit_sequence(label, |q, _, _| { + s.edit_sequence_as(label, merge, |q, _, _| { let (_, it) = q.find_item_mut(clip).ok_or(filmcraft_edit::EditError::NoItem(clip))?; let mt = it.source_time_at(tl.clamp(it.start, it.end() - Tick(1))); let e = it.effects.get_mut(eidx).ok_or_else(|| bad("graphics.set", "no such layer"))?; @@ -744,7 +753,7 @@ fn menu_commands() -> Vec { props.insert("sides".into(), json!(n)); let ph = s.playhead(); let before = s.history.undo.len(); - set_props(s, clip, ei, &props, ph, "Change Graphic Property")?; + set_props(s, clip, ei, &props, ph, "Change Graphic Property", None)?; if s.history.undo.len() > before { s.history.undo.pop(); } @@ -963,7 +972,7 @@ pub fn commands() -> Vec { let mut props = serde_json::Map::new(); props.insert("text".into(), Value::String(text)); let ph = s.playhead(); - set_props(s, clip, ei, &props, ph, "Edit Text")?; + set_props(s, clip, ei, &props, ph, "Edit Text", None)?; if merge && s.history.undo.len() >= 2 { // keep the snapshot from before the typing session s.history.undo.pop(); @@ -976,14 +985,19 @@ pub fn commands() -> Vec { "Set Graphic Properties", &[], None, - r##"{"clip":id?,"layer":n|name?,"props":{"font":"Inter","font_style":"Bold","size":120,"align":"center","tracking":50,"leading":0,"fill_color":"#ffcc00","stroke":true,"stroke_width":6,"background":true,"shadow":true,"position":[x,y],"scale":100,"rotation":0,"opacity":100,…},"time":ticks?}"##, + r##"{"clip":id?,"layer":n|name?,"props":{"font":"Inter","font_style":"Bold","size":120,"align":"center","tracking":50,"leading":0,"fill_color":"#ffcc00","stroke":true,"stroke_width":6,"background":true,"shadow":true,"position":[x,y],"scale":100,"rotation":0,"opacity":100,…},"time":ticks?,"merge":bool?,"begin":bool?}"##, has_graphic, |s, p| { let clip = target_clip(s, p).ok_or_else(|| bad("graphics.set", "no graphic clip"))?; let (_, ei) = layer_effect_index(s, clip, p)?; let props = p.get("props").and_then(Value::as_object).ok_or_else(|| bad("graphics.set", "need `props`"))?.clone(); let tl = time_p(s, p, "").unwrap_or_else(|| s.playhead()); - set_props(s, clip, ei, &props, tl, "Change Graphic Property")?; + // `merge`: a drag of a property is one undo step; `begin` starts a new one (#201) + let merge = bool_p(p, "merge").unwrap_or(false).then(|| format!("graphics.set:{}:{ei}", clip.0)); + if bool_p(p, "begin").unwrap_or(false) { + s.history.merge_key = None; + } + set_props(s, clip, ei, &props, tl, "Change Graphic Property", merge.as_deref())?; Ok(Value::Null) }, ), diff --git a/crates/engine/src/graphics_tests.rs b/crates/engine/src/graphics_tests.rs index 70ede0ced..94589f911 100644 --- a/crates/engine/src/graphics_tests.rs +++ b/crates/engine/src/graphics_tests.rs @@ -153,6 +153,38 @@ fn typing_coalesces_into_one_undo_step_and_props_set() { assert_eq!(text_of(&layers(&s, clip)[0]), ""); } +/// A drag of a property is one undo step: `graphics.set` with `merge` folds consecutive calls +/// into the step the first change of the press began (`begin`), so one undo restores the +/// original value (the color picker in Essential Graphics was committing one step per frame). +#[test] +fn a_property_drag_is_one_undo_step() { + let mut s = demo(); + let r = s.execute("graphics.newText", json!({"text": "Title"})).unwrap(); + let clip = ClipId(r["clip"].as_u64().unwrap()); + s.execute("graphics.set", json!({"clip": clip.0, "layer": 0, "props": {"shadow_color": [0.0, 0.0, 0.0, 1.0]}})).unwrap(); + let color = |s: &Session| -> [f32; 4] { + let q = s.active_sequence().unwrap(); + let (_, it) = q.find_item(clip).unwrap(); + let e = it.effects.iter().find(|e| e.effect == "graphic_text").unwrap(); + e.param("shadow_color").unwrap().value_at(Tick::ZERO).as_color().unwrap() + }; + let before = s.history.undo.len(); + // the drag: the first change of the press begins the step, the rest fold into it + for (i, c) in [[0.1, 0.0, 0.0, 1.0], [0.2, 0.0, 0.0, 1.0], [0.3, 0.0, 0.0, 1.0], [0.4, 0.0, 0.0, 1.0]].iter().enumerate() { + s.execute("graphics.set", json!({"clip": clip.0, "layer": 0, "props": {"shadow_color": c}, "merge": true, "begin": i == 0})).unwrap(); + } + assert_eq!(s.history.undo.len(), before + 1, "one undo step for the whole drag"); + assert_eq!(color(&s)[0], 0.4); + // a second drag (a new press) begins its own step + s.execute("graphics.set", json!({"clip": clip.0, "layer": 0, "props": {"shadow_color": [1.0, 0.0, 0.0, 1.0]}, "merge": true, "begin": true})).unwrap(); + s.execute("graphics.set", json!({"clip": clip.0, "layer": 0, "props": {"shadow_color": [0.0, 1.0, 0.0, 1.0]}, "merge": true})).unwrap(); + assert_eq!(s.history.undo.len(), before + 2, "a new press begins a new step"); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(color(&s)[0], 0.4, "undo takes back only the second drag"); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(color(&s)[0], 0.0, "and then the first"); +} + #[test] fn align_and_distribute_layers() { let mut s = demo(); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index d47c76413..eea55b32a 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -29,6 +29,7 @@ pub mod media_browser; pub mod media_pool; pub mod mixer; pub mod multicam; +pub mod narration; pub mod panels; pub mod perf; pub mod presets; @@ -261,6 +262,10 @@ pub struct EditorState { /// Selected captions (caption tracks / Captions panel). #[serde(default)] pub caption_selection: Vec, + /// Selected transitions (clicked in the Timeline): Effect Controls shows one, Delete removes + /// them. Selecting clips clears it and selecting transitions clears the clip selection. + #[serde(default)] + pub transition_selection: Vec, /// Selected layers (indices among the graphic layers, 0 = back) of the selected graphic clip. #[serde(default)] pub graphic_layers: Vec, @@ -349,6 +354,10 @@ pub struct Session { pub mcrec: multicam::Recorder, /// Voice-over recording: the input device and the take in progress. pub voiceover: voiceover::VoiceOver, + /// The last `tts.preview` result, for the host to play (Text to Speech ▸ Preview). + pub tts_preview: Option>, + /// Synthesized narrations (`tts.render` fills it from a background job). + pub tts_cache: narration::SynthCache, /// Dynamic (J/K/L) trimming and trim-mode loop playback in progress. pub trim_play: trim::TrimPlayback, /// Keyboard shortcuts (active bindings, presets; `shortcuts.*` commands). @@ -451,6 +460,8 @@ impl Session { mixrec: Default::default(), mcrec: Default::default(), voiceover: Default::default(), + tts_preview: None, + tts_cache: Default::default(), trim_play: Default::default(), shortcuts: shortcuts::Shortcuts::new(), offline: Default::default(), @@ -1065,6 +1076,8 @@ mod mixer_tests; #[cfg(test)] mod multicam_tests; #[cfg(test)] +mod narration_tests; +#[cfg(test)] mod nest_editing_tests; #[cfg(test)] mod nest_fidelity_tests; diff --git a/crates/engine/src/narration.rs b/crates/engine/src/narration.rs new file mode 100644 index 000000000..cc9f1fdf8 --- /dev/null +++ b/crates/engine/src/narration.rs @@ -0,0 +1,533 @@ +//! Text to Speech (narrations): write what the narrator says, pick a voice, and get an audio clip. +//! +//! | command | does | +//! |---|---| +//! | `tts.voices` | languages, voices, vocal pitches, pace limits and the pause marker | +//! | `tts.preview` | synthesize without changing the project (`sample: true` speaks the voice's sample sentence); the host plays [`Session::tts_preview`]; `path` also writes a WAV | +//! | `tts.create` | synthesize, write `Narration .wav`, import it and place it at the playhead (one undo step) | +//! | `tts.edit` | change a narration clip's script or voice settings and regenerate it in place (one undo step) | +//! | `tts.inspect` | the narration settings of a clip or item | +//! | `tts.render` | synthesize in a background job into the cache, so the UI's create / edit / preview return at once | +//! | `tts.downloadVoices` | download the natural voices (Kokoro-82M, 9 US English voices, CMUdict; 336 MB, SHA-256 checked) as a background job | +//! +//! **Where a new narration goes.** At `time` (default: the playhead) on `track` if given (it must +//! be free there), else on the first targeted audio track that is free for the narration's length, +//! else the first free audio track, else on a new audio track. Existing clips are never overwritten. +//! +//! **Editing keeps the clip's length** (user decision, 2026-10-07). The clip keeps its start, +//! duration and speed. When the new speech is longer than the clip, the clip still ends where it +//! did but the file holds the whole speech, so trimming the clip's end out reveals the rest. When it +//! is shorter, the file is padded with silence to the clip's length. The clip's source In is reset +//! to the start of the new speech. Other clips (including other clips of the old narration) are not +//! changed. Each edit writes a new file; the old item and its narration stay, so undo is exact. +//! +//! Files are named `Narration .wav` (mono 32-bit float at the voice's rate, 24 kHz) and saved +//! where voice-over recordings go (Scratch Disks ▸ Captured, else next to the project, else +//! `Narrations` in the data or temporary directory); an edit writes next to the previous file. + +use std::sync::{Arc, Mutex, PoisonError}; + +use serde_json::{Value, json}; + +use filmcraft_project::{ClipId, ItemId, ItemKind, MediaRef, Narration, TrackId, TrackKind, VocalPitch}; +use filmcraft_time::{Tick, TimeRange}; +use filmcraft_tts as tts; + +use crate::commands::{CommandSpec, always, bad, has_seq, str_p, time_p, u64_p}; +use crate::{EngineError, Result, Session}; + +/// Item name prefix of generated narrations. +const BASE_NAME: &str = "Narration"; + +/// The narration of a project item. +pub fn narration_of(s: &Session, item: ItemId) -> Option<&Narration> { + s.project.narrations.get(&item) +} + +/// The narration clip a command works on: `clip`, else the first selected clip whose item is a +/// narration. +pub fn target_clip(s: &Session, p: &Value) -> Option { + let seq = s.active_sequence()?; + let is_narration = |c: ClipId| seq.find_item(c).is_some_and(|(_, it)| s.project.narrations.contains_key(&it.item)); + match u64_p(p, "clip") { + Some(c) => Some(ClipId(c)).filter(|c| is_narration(*c)), + None => s.state.selection.iter().copied().find(|c| is_narration(*c)), + } +} + +fn has_narration_clip(s: &Session) -> std::result::Result<(), String> { + has_seq(s)?; + target_clip(s, &Value::Null).map(|_| ()).ok_or_else(|| "select a narration clip".into()) +} + +/// Script and settings from params, on top of `base`. +fn settings(cmd: &str, p: &Value, base: Option<&Narration>) -> Result { + // a setting of the wrong type is an error, never silently replaced by a default + for key in ["text", "voice", "pitch"] { + if p.get(key).is_some_and(|v| !v.is_string()) { + return Err(bad(cmd, format!("`{key}` must be a string"))); + } + } + let text = match str_p(p, "text") { + Some(t) => t.to_string(), + None => base.map(|b| b.text.clone()).ok_or_else(|| bad(cmd, "`text` is required"))?, + }; + let voice = str_p(p, "voice").map(str::to_string).or_else(|| base.map(|b| b.voice.clone())).unwrap_or_else(|| tts::default_voice_id().to_string()); + let info = tts::voices_in(None).into_iter().find(|v| v.id == voice).ok_or_else(|| bad(cmd, format!("unknown voice `{voice}` (see tts.voices)")))?; + let pitch = match str_p(p, "pitch") { + Some(v) => VocalPitch::from_id(v).ok_or_else(|| bad(cmd, format!("unknown pitch `{v}` (extraLow, low, default, high, extraHigh)")))?, + None => base.map(|b| b.pitch).unwrap_or_default(), + }; + let pace = match p.get("pace") { + Some(v) => v.as_f64().ok_or_else(|| bad(cmd, "`pace` must be a number"))?, + None => base.map_or(1.0, |b| b.pace), + }; + let n = Narration { text, language: info.language.to_string(), voice, pitch, pace, speech_duration: Tick::ZERO }; + n.check().map_err(|e| bad(cmd, e))?; + Ok(n) +} + +/// Synthesized narrations by settings, newest last (`tts.render` fills it in the background so +/// the UI's create / edit / preview don't block). +pub type SynthCache = Arc)>>>; +const CACHE_ENTRIES: usize = 6; + +fn cache_key(n: &Narration) -> String { + format!("{}\u{1f}{}\u{1f}{}\u{1f}{}", n.voice, n.pitch.id(), n.pace.to_bits(), n.text) +} + +fn tts_err(cmd: &str, e: tts::TtsError) -> EngineError { + match e { + tts::TtsError::NotInstalled => bad(cmd, "the natural voices are not downloaded yet: Text to Speech ▸ Download natural voices (tts.downloadVoices)"), + e => bad(cmd, e.to_string()), + } +} + +/// Synthesize `n` (no cache). +fn render(n: &Narration, models_dir: Option<&std::path::Path>) -> std::result::Result { + let voice = tts::voice_in(&n.voice, models_dir)?; + voice.synthesize(&n.text, &tts::Params { semitones: n.pitch.semitones(), pace: n.pace }) +} + +fn synthesize(s: &Session, cmd: &str, n: &Narration) -> Result> { + let key = cache_key(n); + if let Some(a) = s.tts_cache.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(k, _)| *k == key).map(|(_, a)| a.clone()) { + return Ok(a); + } + let a = Arc::new(render(n, crate::transcript::models_dir().as_deref()).map_err(|e| tts_err(cmd, e))?); + cache_put(&s.tts_cache, key, a.clone()); + Ok(a) +} + +fn cache_put(c: &SynthCache, key: String, a: Arc) { + let mut c = c.lock().unwrap_or_else(PoisonError::into_inner); + c.retain(|(k, _)| *k != key); + c.push((key, a)); + let extra = c.len().saturating_sub(CACHE_ENTRIES); + c.drain(..extra); +} + +fn package_json() -> Value { + let dir = crate::transcript::models_dir(); + json!({ + "id": tts::catalog::PACKAGE_ID, + "name": tts::catalog::PACKAGE_NAME, + "installed": dir.as_deref().is_some_and(tts::catalog::installed), + "size": tts::catalog::size(), + "missingBytes": dir.as_deref().map_or(tts::catalog::size(), tts::catalog::missing_bytes), + "license": tts::catalog::LICENSE, + "source": tts::catalog::SOURCE, + "available": cfg!(feature = "neural-voices"), + }) +} + +fn voices(_s: &mut Session, _p: &Value) -> Result { + let dir = crate::transcript::models_dir(); + Ok(json!({ + "package": package_json(), + "languages": tts::LANGUAGES.iter().map(|(id, name)| json!({"id": id, "name": name})).collect::>(), + "voices": tts::voices_in(dir.as_deref()), + "defaultVoice": tts::default_voice_id(), + "pitches": VocalPitch::ALL.iter().map(|p| json!({"id": p.id(), "label": p.label(), "semitones": p.semitones()})).collect::>(), + "pace": {"min": tts::MIN_PACE, "max": tts::MAX_PACE, "default": 1.0}, + "pauseMarker": tts::script::PAUSE_MARKER, + "maxTextBytes": tts::MAX_TEXT_BYTES, + "sampleSentence": tts::SAMPLE_SENTENCE, + })) +} + +fn preview(s: &mut Session, p: &Value) -> Result { + let cmd = "tts.preview"; + let base = target_clip(s, p).and_then(|c| s.active_sequence()?.find_item(c).map(|(_, it)| it.item)).and_then(|i| s.project.narrations.get(&i).cloned()); + let mut q = p.clone(); + if q.get("sample").and_then(Value::as_bool).unwrap_or(false) { + q["text"] = json!(tts::SAMPLE_SENTENCE); + } + let n = settings(cmd, &q, base.as_ref())?; + let audio = synthesize(s, cmd, &n)?; + let seconds = audio.seconds(); + let samples = audio.samples.len(); + let rate = audio.sample_rate; + let mut path = Value::Null; + if let Some(dest) = str_p(p, "path") { + let bytes = crate::voiceover::write_wav_f32(&audio.samples, rate); + s.services.write_file(dest, &bytes).map_err(|e| EngineError::Other(format!("{dest}: {e}")))?; + path = json!(dest); + } + s.tts_preview = Some(audio); + Ok(json!({"seconds": seconds, "samples": samples, "sampleRate": rate, "voice": n.voice, "path": path})) +} + +/// Write `samples` as a new WAV and import it (its own undo step; callers collapse history). +/// `near`: an edit writes next to the narration's previous file unless `dir` is given. +fn write_and_import(s: &mut Session, p: &Value, samples: &[f32], rate: u32, near: Option) -> Result<(ItemId, String)> { + let beside = near.filter(|_| p.get("dir").is_none()).and_then(|i| match &s.project.items.get(&i)?.kind { + ItemKind::Media(m) => match &m.media { + MediaRef::File { path } => std::path::Path::new(path).parent().filter(|d| !d.as_os_str().is_empty()).map(|d| d.to_string_lossy().into_owned()), + MediaRef::Generator(_) => None, + }, + _ => None, + }); + let dir = beside.unwrap_or_else(|| crate::voiceover::media_dir(s, p, "Narrations")); + let path = crate::voiceover::unique_wav_path(s, &dir, BASE_NAME)?; + if !cfg!(target_arch = "wasm32") { + std::fs::create_dir_all(&dir).map_err(|e| EngineError::Other(format!("{dir}: {e}")))?; + } + let bytes = crate::voiceover::write_wav_f32(samples, rate); + s.services.write_file(&path, &bytes).map_err(|e| EngineError::Other(format!("{path}: {e}")))?; + let item = crate::commands::import_bytes(s, &path, bytes.into(), None)?; + Ok((item, path)) +} + +fn track_is_free(s: &Session, track: TrackId, range: TimeRange) -> bool { + s.active_sequence() + .and_then(|q| q.audio_tracks.iter().find(|t| t.id == track)) + .is_some_and(|t| !t.locked && !t.items.iter().any(|it| it.start < range.end() && range.start < it.end())) +} + +/// The track a new narration of `range` goes to (None: add a track). +fn placement(s: &Session, p: &Value, range: TimeRange) -> Result> { + let cmd = "tts.create"; + let seq = s.active_sequence().ok_or(EngineError::NoSequence)?; + if let Some(v) = p.get("track") { + let id = + crate::mixer::strip_ref(seq, v).filter(|id| seq.audio_tracks.iter().any(|t| t.id == *id)).ok_or_else(|| bad(cmd, format!("no audio track {v}")))?; + if !track_is_free(s, id, range) { + return Err(bad(cmd, format!("audio track {v} is locked or has clips there"))); + } + return Ok(Some(id)); + } + let targeted = s.targeting().targeted; + let order = seq.audio_tracks.iter().filter(|t| targeted.contains(&t.id)).chain(seq.audio_tracks.iter().filter(|t| !targeted.contains(&t.id))); + Ok(order.map(|t| t.id).find(|id| track_is_free(s, *id, range))) +} + +fn create(s: &mut Session, p: &Value) -> Result { + let cmd = "tts.create"; + let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?; + let mut n = settings(cmd, p, None)?; + let at = time_p(s, p, "").unwrap_or(s.playhead()); + if at < Tick::ZERO { + return Err(bad(cmd, "the start time cannot be negative")); + } + let audio = synthesize(s, cmd, &n)?; + let rate = audio.sample_rate.max(1); + let dur = Tick::from_units(audio.samples.len() as i64, i64::from(rate)); + n.speech_duration = dur; + let range = TimeRange::new(at, dur); + let track = placement(s, p, range)?; + let n0 = s.history.undo.len(); + let result = (|| -> Result { + let track = match track { + Some(t) => t, + None => { + let before: Vec = s.active_sequence().map(|q| q.audio_tracks.iter().map(|t| t.id).collect()).unwrap_or_default(); + s.execute("sequence.addTracks", json!({"video": 0, "audio": 1, "audioAfter": "last"}))?; + s.active_sequence() + .and_then(|q| q.audio_tracks.iter().map(|t| t.id).find(|id| !before.contains(id))) + .ok_or_else(|| EngineError::Other("could not add an audio track".into()))? + } + }; + let (item, path) = write_and_import(s, p, &audio.samples, rate, None)?; + let seq_rate = s.project.sequence(seq_id).map(|q| q.settings.frame_rate).unwrap_or_default(); + let narration = n.clone(); + let clip = s.edit("New Narration", |pr, st| { + let mut it = pr + .make_track_item(item, TrackKind::Audio, at, TimeRange::new(Tick::ZERO, dur), seq_rate) + .ok_or(EngineError::Other("the narration is not importable".into()))?; + it.duration = dur; + let id = it.id; + let seq = pr.sequence_mut(seq_id).ok_or(EngineError::NoSequence)?; + let t = seq.audio_tracks.iter_mut().find(|t| t.id == track).ok_or_else(|| bad(cmd, "the narration track was deleted"))?; + let pos = t.items.partition_point(|x| x.start <= at); + t.items.insert(pos, it); + seq.check().map_err(EngineError::Other)?; + pr.narrations.insert(item, narration); + st.selection = vec![id]; + Ok(id) + })?; + let seq = s.project.sequence(seq_id).ok_or(EngineError::NoSequence)?; + Ok(json!({ + "item": item.0, + "clip": clip.0, + "track": crate::mixer::strip_label(seq, track), + "start": at.0, + "duration": dur.0, + "speechDuration": dur.0, + "path": path, + "sampleRate": rate, + })) + })(); + crate::clip_ops::collapse_history(s, n0, "New Narration"); + if result.is_err() && s.history.undo.len() > n0 { + // a step failed after the track was added or the file imported: roll back to before + let _ = s.undo(); + s.history.redo.clear(); + } + result +} + +/// Samples of media a clip of `duration` at `speed` plays, rounded up (exact for 100 %). +fn needed_samples(duration: Tick, speed: f64, rate: u32) -> usize { + let rate = i64::from(rate.max(1)); + let media = if speed.is_finite() && speed != 0.0 && (speed.abs() - 1.0).abs() > 1e-12 { + // speed is a ratio, not edit time: scale the exact tick count + Tick::from_seconds_f64(duration.seconds() * speed.abs()) + } else { + duration + }; + let mut n = media.to_units_floor(rate); + if Tick::from_units(n, rate) < media { + n = n.saturating_add(1); + } + let cap = (tts::MAX_SECONDS * 2.0) as i64 * rate; + n.clamp(0, cap) as usize +} + +fn edit(s: &mut Session, p: &Value) -> Result { + let cmd = "tts.edit"; + let seq_id = s.state.active_sequence.ok_or(EngineError::NoSequence)?; + let clip = target_clip(s, p).ok_or_else(|| bad(cmd, "select a narration clip (or pass `clip`)"))?; + let seq = s.active_sequence().ok_or(EngineError::NoSequence)?; + let (track, it) = seq.find_item(clip).ok_or_else(|| bad(cmd, "no such clip"))?; + if seq.audio_tracks.iter().any(|t| t.id == track && t.locked) { + return Err(bad(cmd, "the clip's track is locked")); + } + let (old_item, clip_dur, speed) = (it.item, it.duration, it.speed); + let base = s.project.narrations.get(&old_item).cloned().ok_or_else(|| bad(cmd, "the clip is not a narration"))?; + let mut n = settings(cmd, p, Some(&base))?; + let audio = synthesize(s, cmd, &n)?; + let rate = audio.sample_rate.max(1); + let speech = audio.samples.len(); + n.speech_duration = Tick::from_units(speech as i64, i64::from(rate)); + // the clip keeps its length: pad the file with silence to what the clip plays + let needed = needed_samples(clip_dur, speed, rate); + let mut samples = audio.samples.clone(); + if needed > samples.len() { + samples.try_reserve(needed - samples.len()).map_err(|e| bad(cmd, format!("unable to allocate the narration: {e}")))?; + samples.resize(needed, 0.0); + } + let file_len = samples.len(); + let n0 = s.history.undo.len(); + let result = (|| -> Result { + let (item, path) = write_and_import(s, p, &samples, rate, Some(old_item))?; + let name = s.project.items.get(&item).map(|i| i.name.clone()).unwrap_or_default(); + let narration = n.clone(); + s.edit("Edit Narration", |pr, _| { + let seq = pr.sequence_mut(seq_id).ok_or(EngineError::NoSequence)?; + let (_, it) = seq.find_item_mut(clip).ok_or_else(|| bad(cmd, "the clip was deleted"))?; + it.item = item; + it.name = name; + it.source_in = Tick::ZERO; + seq.check().map_err(EngineError::Other)?; + pr.narrations.insert(item, narration); + Ok(()) + })?; + Ok(json!({ + "clip": clip.0, + "item": item.0, + "previousItem": old_item.0, + "duration": clip_dur.0, + "speechDuration": n.speech_duration.0, + "fileDuration": Tick::from_units(file_len as i64, i64::from(rate)).0, + "path": path, + "sampleRate": rate, + })) + })(); + crate::clip_ops::collapse_history(s, n0, "Edit Narration"); + if result.is_err() && s.history.undo.len() > n0 { + let _ = s.undo(); + s.history.redo.clear(); + } + result +} + +fn inspect(s: &mut Session, p: &Value) -> Result { + let cmd = "tts.inspect"; + let (item, clip) = match u64_p(p, "item") { + Some(i) => (ItemId(i), None), + None => { + let c = target_clip(s, p).ok_or_else(|| bad(cmd, "pass `item` or `clip`, or select a narration clip"))?; + let seq = s.active_sequence().ok_or(EngineError::NoSequence)?; + (seq.find_item(c).map(|(_, it)| it.item).ok_or_else(|| bad(cmd, "no such clip"))?, Some(c.0)) + } + }; + let n = s.project.narrations.get(&item).ok_or_else(|| bad(cmd, "not a narration"))?; + Ok(json!({"item": item.0, "clip": clip, "narration": n})) +} + +fn new_job(s: &mut Session, label: String) -> crate::Job { + let id = s.jobs.iter().map(|j| j.id).max().unwrap_or(0) + 1; + crate::Job { id, label, progress: Default::default(), result: Default::default() } +} + +fn spawn(s: &mut Session, job: crate::Job, wait: bool, name: &str, run: impl FnOnce() + Send + 'static) -> Result { + let id = job.id; + let run = crate::export_tools::guard_job(job.progress.clone(), job.result.clone(), run); + s.jobs.push(job); + if wait || cfg!(target_arch = "wasm32") { + run(); + } else { + std::thread::Builder::new().name(name.into()).spawn(run).map_err(|e| EngineError::Other(e.to_string()))?; + } + Ok(id) +} + +type JobResult = std::sync::Mutex>>; + +fn finish(prog: &filmcraft_export::Progress, res: &JobResult, r: std::result::Result, t0: web_time::Instant) { + use std::sync::atomic::Ordering; + let secs = t0.elapsed().as_secs_f64(); + let r = match r { + Ok(status) => { + *prog.status.lock().unwrap_or_else(PoisonError::into_inner) = status; + Ok(filmcraft_export::Report { path: String::new(), frames: 0, seconds: secs, bytes: 0, render_fps: 0.0, extra_files: Vec::new() }) + } + Err(e) => { + *prog.status.lock().unwrap_or_else(PoisonError::into_inner) = e.clone(); + *prog.error.lock().unwrap_or_else(PoisonError::into_inner) = Some(e.clone()); + Err(e) + } + }; + prog.finished.store(true, Ordering::Relaxed); + *res.lock().unwrap_or_else(PoisonError::into_inner) = Some(r); +} + +/// Synthesize in the background into the cache, so `tts.create` / `tts.edit` / `tts.preview` with +/// the same settings return at once. +fn render_job(s: &mut Session, p: &Value) -> Result { + let cmd = "tts.render"; + let base = target_clip(s, p).and_then(|c| s.active_sequence()?.find_item(c).map(|(_, it)| it.item)).and_then(|i| s.project.narrations.get(&i).cloned()); + let mut q = p.clone(); + if q.get("sample").and_then(Value::as_bool).unwrap_or(false) { + q["text"] = json!(tts::SAMPLE_SENTENCE); + } + let n = settings(cmd, &q, base.as_ref())?; + let key = cache_key(&n); + if s.tts_cache.lock().unwrap_or_else(PoisonError::into_inner).iter().any(|(k, _)| *k == key) { + return Ok(json!({"job": null, "cached": true})); + } + let job = new_job(s, "Synthesizing narration".into()); + let (prog, res, cache) = (job.progress.clone(), job.result.clone(), s.tts_cache.clone()); + let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false); + let id = spawn(s, job, wait, "filmcraft-tts", move || { + let t0 = web_time::Instant::now(); + *prog.status.lock().unwrap_or_else(PoisonError::into_inner) = "Synthesizing…".into(); + let r = match render(&n, crate::transcript::models_dir().as_deref()) { + Ok(a) => { + let secs = a.seconds(); + cache_put(&cache, key, Arc::new(a)); + Ok(format!("{secs:.1} s of speech")) + } + Err(tts::TtsError::NotInstalled) => Err("the natural voices are not downloaded yet".to_string()), + Err(e) => Err(e.to_string()), + }; + finish(&prog, &res, r, t0); + })?; + Ok(json!({"job": id, "cached": false})) +} + +/// Download the natural voices (Kokoro-82M, its voices and CMUdict) in the background. +fn download_voices(s: &mut Session, p: &Value) -> Result { + let cmd = "tts.downloadVoices"; + let dir = crate::transcript::models_dir().ok_or_else(|| bad(cmd, "no data directory for voice models"))?; + if tts::catalog::installed(&dir) { + return Ok(json!({"job": null, "installed": true})); + } + if !cfg!(feature = "neural-voices") { + return Err(bad(cmd, "natural voices are not available in this build (built without the `neural-voices` feature)")); + } + let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false); + let job = new_job(s, format!("Downloading {}", tts::catalog::PACKAGE_NAME)); + job.progress.total.store(tts::catalog::missing_bytes(&dir), std::sync::atomic::Ordering::Relaxed); + let (prog, res) = (job.progress.clone(), job.result.clone()); + let id = spawn(s, job, wait, "filmcraft-tts-download", move || { + let t0 = web_time::Instant::now(); + let r = download_package(&dir, &prog); + finish(&prog, &res, r.map(|()| "Natural voices installed".to_string()), t0); + })?; + Ok(json!({"job": id, "installed": false})) +} + +#[cfg(feature = "neural-voices")] +fn download_package(dir: &std::path::Path, prog: &filmcraft_export::Progress) -> std::result::Result<(), String> { + use std::sync::atomic::Ordering; + let files: Vec = + tts::catalog::FILES.iter().map(|f| filmcraft_speech::models::ModelFile { name: f.name, url: f.url, sha256: f.sha256, size: f.size }).collect(); + let mut progress = |done: u64, total: u64, file: &str| { + prog.done.store(done, Ordering::Relaxed); + prog.total.store(total, Ordering::Relaxed); + *prog.status.lock().unwrap_or_else(PoisonError::into_inner) = format!("Downloading {file}"); + !prog.cancel.load(Ordering::Relaxed) + }; + filmcraft_speech::models::download_files(&tts::catalog::package_dir(dir), &files, &mut progress).map_err(|e| e.to_string()) +} + +#[cfg(not(feature = "neural-voices"))] +fn download_package(_: &std::path::Path, _: &filmcraft_export::Progress) -> std::result::Result<(), String> { + Err("natural voices are not available in this build".into()) +} + +fn spec( + id: &'static str, + label: &'static str, + params: &'static str, + enabled: fn(&Session) -> std::result::Result<(), String>, + run: fn(&mut Session, &Value) -> Result, + journal: bool, +) -> CommandSpec { + CommandSpec { id, label, menu: &[], shortcut: None, params, enabled, run, journal } +} + +pub fn commands() -> Vec { + vec![ + spec("tts.voices", "List Narration Voices", "{}", always, voices, false), + spec( + "tts.preview", + "Preview Narration", + r#"{"text":str?,"voice":str?,"pitch":"default"?,"pace":f64?,"sample":bool?,"clip":id?,"path":str?}"#, + always, + preview, + false, + ), + spec( + "tts.create", + "New Narration", + r#"{"text":str,"voice":str?,"pitch":"extraLow|low|default|high|extraHigh"?,"pace":0.5..2?,"time":ticks?,"track":"A1"|id?,"dir":str?}"#, + has_seq, + create, + true, + ), + spec("tts.edit", "Edit Narration", r#"{"clip":id?,"text":str?,"voice":str?,"pitch":str?,"pace":f64?,"dir":str?}"#, has_narration_clip, edit, true), + spec("tts.inspect", "Inspect Narration", r#"{"clip":id?,"item":id?}"#, always, inspect, false), + spec( + "tts.render", + "Synthesize Narration", + r#"{"text":str?,"voice":str?,"pitch":str?,"pace":f64?,"sample":bool?,"clip":id?,"wait":bool?}"#, + always, + render_job, + false, + ), + spec("tts.downloadVoices", "Download Natural Voices", r#"{"wait":bool?}"#, always, download_voices, false), + ] +} diff --git a/crates/engine/src/narration_tests.rs b/crates/engine/src/narration_tests.rs new file mode 100644 index 000000000..21896b5e8 --- /dev/null +++ b/crates/engine/src/narration_tests.rs @@ -0,0 +1,425 @@ +//! Tests of [`crate::narration`]: Text to Speech narrations are placed without overwriting, keep +//! their clip length when edited, and are one exact undo step. + +use super::*; +use filmcraft_project::{ClipId, ItemId}; +use serde_json::json; + +const RATE: i64 = filmcraft_tts::SAMPLE_RATE as i64; + +fn tmp(name: &str) -> String { + let d = std::env::temp_dir().join(format!("filmcraft-tts-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + d.to_string_lossy().into_owned() +} + +fn demo() -> Session { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s +} + +/// The 32-bit float samples of a WAV written by `write_wav_f32`. +fn wav_samples(path: &str) -> Vec { + let b = std::fs::read(path).unwrap(); + b[44..].as_chunks::<4>().0.iter().map(|c| f32::from_le_bytes(*c)).collect() +} + +fn clip(s: &Session, id: u64) -> filmcraft_project::TrackItem { + s.active_sequence().unwrap().find_item(ClipId(id)).unwrap().1.clone() +} + +fn all_clips(s: &Session) -> Vec { + s.active_sequence().unwrap().all_tracks().flat_map(|t| t.items.clone()).collect() +} + +#[test] +fn voices_lists_the_built_in_voices() { + let mut s = Session::default(); + let r = s.execute("tts.voices", json!({})).unwrap(); + let ids: Vec<&str> = r["voices"].as_array().unwrap().iter().map(|v| v["id"].as_str().unwrap()).collect(); + assert!(ids.contains(&"basic-female") && ids.contains(&"basic-male"), "{ids:?}"); + assert_eq!(r["languages"][0]["id"], "en-US"); + assert_eq!(r["pitches"].as_array().unwrap().len(), 5); + assert_eq!(r["pauseMarker"], "[pause 1s]"); +} + +#[test] +fn create_places_the_narration_at_the_playhead_as_one_undo_step() { + let mut s = demo(); + let dir = tmp("create"); + s.execute("playhead.set", json!({"seconds": 2.0})).unwrap(); + let t0 = s.playhead(); + assert!(t0 > Tick::ZERO); + let before = all_clips(&s); + let undo0 = s.history.undo.len(); + let r = s.execute("tts.create", json!({"text": "Welcome to FilmCraft.", "voice": "basic-male", "pitch": "low", "pace": 1.2, "dir": dir})).unwrap(); + let c = clip(&s, r["clip"].as_u64().unwrap()); + assert_eq!(c.start, t0); + // the clip is exactly as long as the speech (sample-exact) and plays it from the start + let samples = wav_samples(r["path"].as_str().unwrap()); + assert_eq!(c.duration, Tick::from_units(samples.len() as i64, RATE)); + assert_eq!(c.source_in, Tick::ZERO); + assert_eq!(r["speechDuration"], c.duration.0); + let n = s.project.narrations.get(&ItemId(r["item"].as_u64().unwrap())).unwrap(); + assert_eq!((n.text.as_str(), n.voice.as_str(), n.pitch, n.pace), ("Welcome to FilmCraft.", "basic-male", VocalPitch::Low, 1.2)); + assert_eq!(n.speech_duration, c.duration); + assert_eq!(s.state.selection, vec![c.id]); + // nothing that was there before moved or changed + let after = all_clips(&s); + for b in &before { + assert!(after.contains(b), "clip {:?} changed", b.id); + } + assert_eq!(after.len(), before.len() + 1); + // one undo step that removes the clip and the narration; redo brings both back + assert_eq!(s.history.undo.len(), undo0 + 1); + assert_eq!(s.history.undo.last().unwrap().0, "New Narration"); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(all_clips(&s), before); + assert!(s.project.narrations.is_empty()); + s.execute("edit.redo", json!({})).unwrap(); + assert_eq!(clip(&s, c.id.0), c); + assert_eq!(s.project.narrations.len(), 1); + let _ = std::fs::remove_dir_all(&dir); +} + +use filmcraft_project::VocalPitch; +use filmcraft_time::Tick; + +#[test] +fn create_never_overwrites_and_adds_a_track_when_every_track_is_busy() { + let mut s = demo(); + let dir = tmp("busy"); + // fill every audio track over 0–60 s with a narration-length clip so none is free at 0 + let tracks = s.active_sequence().unwrap().audio_tracks.len(); + let first = s.execute("tts.create", json!({"text": "One two three.", "time": 0, "dir": dir})).unwrap(); + let busy_len = Tick(first["duration"].as_i64().unwrap()); + for _ in 1..tracks + 2 { + s.execute("tts.create", json!({"text": "One two three.", "time": 0, "dir": dir})).unwrap(); + } + // every create landed on its own track at 0: no two narrations overlap on one track + let seq = s.active_sequence().unwrap(); + for t in &seq.audio_tracks { + let mut items: Vec<_> = t.items.iter().map(|i| (i.start, i.end())).collect(); + items.sort(); + for w in items.windows(2) { + assert!(w[0].1 <= w[1].0, "overlap on {:?}", t.name); + } + } + assert!(seq.audio_tracks.len() > tracks, "a new audio track was added"); + assert!(busy_len > Tick::ZERO); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn create_on_an_occupied_or_unknown_track_is_refused_and_changes_nothing() { + let mut s = demo(); + let dir = tmp("occupied"); + let r = s.execute("tts.create", json!({"text": "Hello.", "time": 0, "track": "A1", "dir": dir})); + if s.active_sequence().unwrap().audio_tracks[0].items.iter().any(|i| i.start <= Tick::ZERO && Tick::ZERO < i.end()) { + assert!(r.is_err(), "A1 is busy at 0"); + } + let undo0 = s.history.undo.len(); + let items0 = s.project.items.len(); + assert!(s.execute("tts.create", json!({"text": "Hello.", "track": "A99", "dir": dir})).is_err()); + s.execute("timeline.setTrack", json!({"track": "A2", "locked": true})).unwrap(); + let undo1 = s.history.undo.len(); + assert!(s.execute("tts.create", json!({"text": "Hello.", "track": "A2", "time": 0, "dir": dir})).is_err()); + assert_eq!(s.history.undo.len(), undo1); + assert!(undo1 >= undo0); + assert_eq!(s.project.items.len(), items0, "no file was imported"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn editing_to_longer_speech_keeps_the_clip_length_and_the_whole_speech() { + let mut s = demo(); + let dir = tmp("longer"); + let r = s.execute("tts.create", json!({"text": "Hi.", "time": 0, "dir": dir})).unwrap(); + let c0 = clip(&s, r["clip"].as_u64().unwrap()); + let undo0 = s.history.undo.len(); + let e = s.execute("tts.edit", json!({"clip": c0.id.0, "text": "Hi. This narration is now a great deal longer than it was before."})).unwrap(); + let c1 = clip(&s, c0.id.0); + assert_eq!((c1.id, c1.start, c1.duration, c1.speed), (c0.id, c0.start, c0.duration, c0.speed), "position and length kept"); + assert_ne!(c1.item, c0.item, "a new file"); + assert_eq!(c1.source_in, Tick::ZERO); + let n = s.project.narrations.get(&c1.item).unwrap(); + assert!(n.speech_duration > c1.duration, "the speech is longer than the clip"); + // the file holds the whole speech, nothing padded + let samples = wav_samples(e["path"].as_str().unwrap()); + assert_eq!(Tick::from_units(samples.len() as i64, RATE), n.speech_duration); + assert_eq!(e["fileDuration"], n.speech_duration.0); + // the old item and its narration are untouched (other clips of it, undo) + assert_eq!(s.project.narrations.get(&c0.item).unwrap().text, "Hi."); + assert_eq!(s.history.undo.len(), undo0 + 1); + assert_eq!(s.history.undo.last().unwrap().0, "Edit Narration"); + // the edited file sits next to the first one + assert!(e["path"].as_str().unwrap().starts_with(&dir), "{}", e["path"]); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn editing_to_shorter_speech_pads_the_file_with_exact_silence() { + let mut s = demo(); + let dir = tmp("shorter"); + let r = s.execute("tts.create", json!({"text": "This first version of the narration is fairly long, with several words.", "time": 0, "dir": dir})).unwrap(); + let id = r["clip"].as_u64().unwrap(); + let c0 = clip(&s, id); + let e = s.execute("tts.edit", json!({"clip": id, "text": "Short."})).unwrap(); + let c1 = clip(&s, id); + assert_eq!((c1.start, c1.duration), (c0.start, c0.duration)); + let n = s.project.narrations.get(&c1.item).unwrap().clone(); + assert!(n.speech_duration < c1.duration); + let samples = wav_samples(e["path"].as_str().unwrap()); + let speech = n.speech_duration.to_units_floor(RATE) as usize; + // the file covers the clip, and everything after the speech is exactly zero + assert!(Tick::from_units(samples.len() as i64, RATE) >= c1.duration); + assert!(samples.len() > speech); + assert!(samples[speech..].iter().all(|&x| x == 0.0)); + assert!(samples[..speech].iter().any(|&x| x != 0.0)); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn undo_and_redo_of_an_edit_are_exact() { + let mut s = demo(); + let dir = tmp("undo"); + let r = s.execute("tts.create", json!({"text": "Before.", "time": 0, "dir": dir})).unwrap(); + let id = r["clip"].as_u64().unwrap(); + let before = (*s.project).clone(); + s.execute("tts.edit", json!({"clip": id, "text": "After.", "voice": "basic-male", "pitch": "extraHigh", "pace": 0.7})).unwrap(); + let after = (*s.project).clone(); + assert_ne!(before, after); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(*s.project, before); + let n = s.project.narrations.get(&clip(&s, id).item).unwrap(); + assert_eq!((n.text.as_str(), n.voice.as_str()), ("Before.", "basic-female")); + s.execute("edit.redo", json!({})).unwrap(); + assert_eq!(*s.project, after); + let n = s.project.narrations.get(&clip(&s, id).item).unwrap(); + assert_eq!((n.text.as_str(), n.pitch, n.pace), ("After.", VocalPitch::ExtraHigh, 0.7)); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn edit_without_changes_keeps_settings_and_settings_inherit() { + let mut s = demo(); + let dir = tmp("inherit"); + let r = s.execute("tts.create", json!({"text": "Keep me.", "voice": "basic-male", "pitch": "high", "pace": 1.5, "time": 0, "dir": dir})).unwrap(); + let id = r["clip"].as_u64().unwrap(); + s.execute("tts.edit", json!({"clip": id, "pace": 0.8})).unwrap(); + let n = s.project.narrations.get(&clip(&s, id).item).unwrap(); + assert_eq!((n.text.as_str(), n.voice.as_str(), n.pitch, n.pace), ("Keep me.", "basic-male", VocalPitch::High, 0.8)); + let i = s.execute("tts.inspect", json!({"clip": id})).unwrap(); + assert_eq!(i["narration"]["pace"], 0.8); + assert_eq!(i["narration"]["pitch"], "high"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn edit_is_disabled_without_a_narration_clip() { + let mut s = demo(); + let dir = tmp("enabled"); + s.state.selection.clear(); + assert!(!s.is_enabled("tts.edit")); + assert!(s.execute("tts.edit", json!({"text": "x"})).is_err()); + // a non-narration clip is refused + let other = s.active_sequence().unwrap().all_tracks().find_map(|t| t.items.first().map(|i| i.id)).unwrap(); + assert!(s.execute("tts.edit", json!({"clip": other.0, "text": "x"})).is_err()); + let r = s.execute("tts.create", json!({"text": "Enable me.", "time": 0, "dir": dir})).unwrap(); + assert!(s.is_enabled("tts.edit"), "the new narration is selected"); + s.state.selection.clear(); + assert!(s.execute("tts.edit", json!({"clip": r["clip"], "text": "By id."})).is_ok()); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn edit_on_a_locked_track_is_refused() { + let mut s = demo(); + let dir = tmp("locked"); + let r = s.execute("tts.create", json!({"text": "Locked.", "time": 0, "dir": dir})).unwrap(); + let track = r["track"].as_str().unwrap().to_string(); + s.execute("timeline.setTrack", json!({"track": track, "locked": true})).unwrap(); + let snap = (*s.project).clone(); + assert!(s.execute("tts.edit", json!({"clip": r["clip"], "text": "Changed."})).is_err()); + assert_eq!(*s.project, snap); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn hostile_parameters_are_refused_without_changes() { + let mut s = demo(); + let dir = tmp("hostile"); + let snap = (*s.project).clone(); + let undo0 = s.history.undo.len(); + let big = "word ".repeat(300_000); + let cases = [ + json!({}), + json!({"text": ""}), + json!({"text": " [pause 2s] "}), + json!({"text": "👍"}), + json!({"text": 42}), + json!({"text": big}), + json!({"text": "hi", "voice": "nope"}), + json!({"text": "hi", "voice": 7}), + json!({"text": "hi", "pitch": "loud"}), + json!({"text": "hi", "pace": "fast"}), + json!({"text": "hi", "pace": 5.0}), + json!({"text": "hi", "pace": -1}), + json!({"text": "hi", "pace": 1e308}), + json!({"text": "hi", "time": -5}), + json!({"text": "hi", "track": {"x": 1}}), + json!({"text": "hi", "track": "V1"}), + ]; + for p in cases { + let mut q = p.clone(); + q["dir"] = json!(dir); + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| s.execute("tts.create", q.clone()))); + assert!(r.is_ok(), "panicked on {p}"); + assert!(r.unwrap().is_err(), "accepted {p}"); + assert_eq!(*s.project, snap, "changed by {p}"); + assert_eq!(s.history.undo.len(), undo0); + } + for p in [json!({"clip": 999_999_999}), json!({"clip": "x"}), json!({"clip": -1})] { + assert!(s.execute("tts.edit", p.clone()).is_err(), "{p}"); + assert!(s.execute("tts.inspect", p.clone()).is_err(), "{p}"); + } + assert!(s.execute("tts.preview", json!({"text": "hi", "pace": 9})).is_err()); + // no sequence open + let mut empty = Session::default(); + assert!(empty.execute("tts.create", json!({"text": "hi", "dir": dir})).is_err()); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn preview_changes_nothing_and_hands_the_audio_to_the_host() { + let mut s = demo(); + let dir = tmp("preview"); + std::fs::create_dir_all(&dir).unwrap(); + let snap = (*s.project).clone(); + let undo0 = s.history.undo.len(); + let path = format!("{dir}/p.wav"); + let r = s.execute("tts.preview", json!({"text": "Preview me.", "path": path})).unwrap(); + assert_eq!(*s.project, snap); + assert_eq!(s.history.undo.len(), undo0); + let a = s.tts_preview.clone().unwrap(); + assert_eq!(r["samples"], a.samples.len()); + assert_eq!(wav_samples(&path), a.samples); + // the sample sentence for "Hear this voice" + let r = s.execute("tts.preview", json!({"sample": true, "voice": "basic-male"})).unwrap(); + assert_eq!(r["voice"], "basic-male"); + assert!(r["seconds"].as_f64().unwrap() > 1.0); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn narrations_survive_save_and_reload() { + let mut s = demo(); + let dir = tmp("save"); + let r = s.execute("tts.create", json!({"text": "Saved [pause 500ms] and loaded.", "pitch": "extraLow", "time": 0, "dir": dir})).unwrap(); + let bytes = filmcraft_format::encode(&s.project, false); + let loaded = filmcraft_format::decode(&bytes).unwrap(); + assert_eq!(loaded.project.narrations, s.project.narrations); + let n = loaded.project.narrations.get(&ItemId(r["item"].as_u64().unwrap())).unwrap(); + assert_eq!((n.text.as_str(), n.pitch), ("Saved [pause 500ms] and loaded.", VocalPitch::ExtraLow)); + // a project without narrations writes no `narrations` key (older files are unchanged) + let plain = String::from_utf8(filmcraft_format::encode(&demo().project, false)).unwrap(); + assert!(!plain.contains("narrations")); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn the_narration_is_heard_in_the_mix() { + let mut s = demo(); + let dir = tmp("mix"); + let r = s.execute("tts.create", json!({"text": "Can you hear me now?", "time": 0, "dir": dir})).unwrap(); + let track = r["track"].as_str().unwrap().to_string(); + let n = s.active_sequence().unwrap().audio_tracks.len(); + for i in 1..=n { + if format!("A{i}") != track { + s.execute("mixer.setStrip", json!({"strip": format!("A{i}"), "muted": true})).unwrap(); + } + } + let provider = s.media.provider(s.project.clone(), s.services.clone()); + let sr = s.active_sequence().unwrap().settings.sample_rate as usize; + let mix = filmcraft_render::audio::mix_sequence(&s.project, s.active_sequence().unwrap(), 0, sr, &provider).channels; + let rms = (mix[0].iter().map(|x| f64::from(x * x)).sum::() / sr as f64).sqrt(); + assert!(rms > 0.01, "narration audible, rms {rms}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn render_fills_the_cache_that_create_and_preview_use() { + let mut s = demo(); + let dir = tmp("render"); + let p = json!({"text": "Rendered in the background.", "voice": "basic-male", "pace": 1.1}); + let r = s.execute("tts.render", json!({"text": "Rendered in the background.", "voice": "basic-male", "pace": 1.1, "wait": true})).unwrap(); + let job = r["job"].as_u64().expect("a job"); + let j = s.jobs.iter().find(|j| j.id == job).unwrap(); + assert!(j.progress.finished.load(std::sync::atomic::Ordering::Relaxed)); + assert!(matches!(&*j.result.lock().unwrap(), Some(Ok(_)))); + // the same settings are now cached: render again is a no-op, preview hands back the same audio + assert_eq!(s.execute("tts.render", p.clone()).unwrap()["cached"], true); + s.execute("tts.preview", p.clone()).unwrap(); + let cached = s.tts_cache.lock().unwrap().last().unwrap().1.clone(); + assert!(std::sync::Arc::ptr_eq(s.tts_preview.as_ref().unwrap(), &cached)); + let mut q = p.clone(); + q["dir"] = json!(dir); + q["time"] = json!(0); + let c = s.execute("tts.create", q).unwrap(); + assert_eq!(c["speechDuration"], Tick::from_units(cached.samples.len() as i64, RATE).0); + // a failing render reports the error in the job + let r = s.execute("tts.render", json!({"text": "x", "voice": "kokoro-heart", "wait": true})); + if let Ok(r) = r { + let job = r["job"].as_u64().unwrap(); + let j = s.jobs.iter().find(|j| j.id == job).unwrap(); + assert!(matches!(&*j.result.lock().unwrap(), Some(Err(_))) || cfg!(feature = "neural-voices")); + } + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn natural_voices_are_listed_and_explain_what_is_missing() { + let mut s = demo(); + let v = s.execute("tts.voices", json!({})).unwrap(); + let ids: Vec<&str> = v["voices"].as_array().unwrap().iter().map(|x| x["id"].as_str().unwrap()).collect(); + assert!(ids.contains(&"kokoro-heart") && ids.contains(&"kokoro-michael"), "{ids:?}"); + assert_eq!(v["package"]["id"], "kokoro-82m"); + assert!(v["package"]["size"].as_u64().unwrap() > 300_000_000); + assert_eq!(v["package"]["available"], cfg!(feature = "neural-voices")); + if !v["package"]["installed"].as_bool().unwrap() { + let e = s.execute("tts.create", json!({"text": "Hi.", "voice": "kokoro-heart"})).unwrap_err().to_string(); + assert!(e.contains("download") || e.contains("not available"), "{e}"); + } + if !cfg!(feature = "neural-voices") && !v["package"]["installed"].as_bool().unwrap() { + let e = s.execute("tts.downloadVoices", json!({})).unwrap_err().to_string(); + assert!(e.contains("not available in this build"), "{e}"); + } +} + +#[test] +fn a_project_subset_keeps_only_the_narrations_of_kept_items() { + let mut s = demo(); + let dir = tmp("subset"); + let r = s.execute("tts.create", json!({"text": "Hello.", "voice": "basic-male", "dir": dir})).unwrap(); + let item = ItemId(r["item"].as_u64().unwrap()); + let others: std::collections::BTreeSet = s.project.items.keys().copied().filter(|i| *i != item).collect(); + assert!(crate::project_tools::project_subset(&s.project, &others).narrations.is_empty()); + let with: std::collections::BTreeSet = [item].into(); + assert!(crate::project_tools::project_subset(&s.project, &with).narrations.contains_key(&item)); +} + +#[test] +fn unique_wav_path_gives_up_instead_of_looping_forever() { + let mut s = demo(); + let first = crate::voiceover::unique_wav_path(&s, "/nonexistent-dir", "Narration").unwrap(); + assert!(first.ends_with("Narration 1.wav"), "{first}"); + // every candidate name taken by a project item: an error, not an endless loop + let template = s.project.items.values().next().cloned().unwrap(); + for k in 1..=99_999u32 { + let mut it = template.clone(); + it.name = format!("Narration {k}.wav"); + std::sync::Arc::make_mut(&mut s.project).items.insert(ItemId(1_000_000 + u64::from(k)), it); + } + assert!(crate::voiceover::unique_wav_path(&s, "/nonexistent-dir", "Narration").is_err()); +} diff --git a/crates/engine/src/project_tools.rs b/crates/engine/src/project_tools.rs index c0390d9dc..a23e7530c 100644 --- a/crates/engine/src/project_tools.rs +++ b/crates/engine/src/project_tools.rs @@ -756,6 +756,7 @@ pub fn project_subset(p: &Project, keep: &BTreeSet) -> Project { out.items.retain(|id, _| keep.contains(id)); prune(&mut out.root, keep); out.transcripts.retain(|id, _| keep.contains(id)); + out.narrations.retain(|id, _| keep.contains(id)); out.search_bins.clear(); out } diff --git a/crates/engine/src/sequence_extras.rs b/crates/engine/src/sequence_extras.rs index ff6cafd63..389590c81 100644 --- a/crates/engine/src/sequence_extras.rs +++ b/crates/engine/src/sequence_extras.rs @@ -428,7 +428,7 @@ fn simplify(s: &mut Session, p: &Value) -> Result { let report = simplify_sequence(pr, &mut q, &o); q.check().map_err(EngineError::Other)?; let bin = pr.root.parent_of(seq_id).filter(|b| b.0 != 0); - let id = pr.add_item(&name, src.label, ItemKind::Sequence(Box::new(q)), bin); + let id = pr.add_item(&name, src.label, ItemKind::Sequence(std::sync::Arc::new(q)), bin); st.active_sequence = Some(id); if !st.open_sequences.contains(&id) { st.open_sequences.push(id); diff --git a/crates/engine/src/settings.rs b/crates/engine/src/settings.rs index bd802fba3..2f020835f 100644 --- a/crates/engine/src/settings.rs +++ b/crates/engine/src/settings.rs @@ -486,6 +486,8 @@ pub struct TimelinePrefs { pub match_frame_sets_in: bool, pub restore_open_sequences: bool, pub add_tracks_automatically: bool, + /// Files dropped onto the Timeline are placed directly on it. + pub drop_import_to_timeline: bool, } impl Default for TimelinePrefs { @@ -512,6 +514,7 @@ impl Default for TimelinePrefs { match_frame_sets_in: false, restore_open_sequences: true, add_tracks_automatically: true, + drop_import_to_timeline: true, } } } @@ -1080,6 +1083,7 @@ static CATEGORIES: &[Category] = &[ b("timeline.matchFrameSetsIn", "Match frame sets in point", false), b("timeline.restoreOpenSequences", "Restore open sequences when opening projects", false), b("timeline.addTracksAutomatically", "Add tracks automatically when editing source clips onto the Timeline", false), + b("timeline.dropImportToTimeline", "Place files dropped onto the Timeline directly on the Timeline", true), ], }, Category { diff --git a/crates/engine/src/settings_tests.rs b/crates/engine/src/settings_tests.rs index 6744a9c34..241f36f6d 100644 --- a/crates/engine/src/settings_tests.rs +++ b/crates/engine/src/settings_tests.rs @@ -552,6 +552,47 @@ fn hardware_decoding_setting_drives_the_decoder_switch() { assert!(settings::field("playback.hardwareDecoding").is_some_and(|f| f.wired && matches!(f.kind, Kind::Choice(_)))); } +/// Settings ▸ Timeline ▸ "Place files dropped onto the Timeline directly on the Timeline": on by default. +#[test] +fn drop_import_to_timeline_defaults_to_on() { + assert!(settings::TimelinePrefs::default().drop_import_to_timeline); + assert!(Preferences::default().timeline.drop_import_to_timeline); +} + +/// The key is saved under `timeline.dropImportToTimeline`, read back by `prefs.get`, and persisted. +#[test] +fn drop_import_to_timeline_round_trips_through_prefs() { + let dir = tmp_dir("drop-import-to-timeline"); + let path = dir.join("preferences.json"); + let mut s = Session { prefs_path: Some(path.clone()), ..Session::default() }; + set(&mut s, "timeline.dropImportToTimeline", json!(false)); + assert!(!s.prefs.timeline.drop_import_to_timeline); + assert_eq!(s.execute("prefs.get", json!({"key": "timeline.dropImportToTimeline"})).unwrap(), json!(false)); + assert!(!Preferences::load(&path).timeline.drop_import_to_timeline); + // a non-boolean is rejected and the stored value is kept + assert!(s.execute("prefs.set", json!({"key": "timeline.dropImportToTimeline", "value": "no"})).is_err()); + assert!(!s.prefs.timeline.drop_import_to_timeline); + std::fs::remove_dir_all(dir).unwrap(); +} + +/// Preferences files written before the key existed load it as on. +#[test] +fn drop_import_to_timeline_defaults_to_on_for_older_prefs() { + assert!(serde_json::from_value::(json!({})).unwrap().drop_import_to_timeline); + let old = Preferences::from_value(json!({"version": 2, "timeline": {"stillImageDuration": 3}})); + assert!(old.timeline.drop_import_to_timeline); + assert_eq!(old.timeline.still_image_duration, 3.0); +} + +/// The key is a Boolean field of the Timeline category in the settings registry. +#[test] +fn drop_import_to_timeline_is_in_the_settings_registry() { + let (cat, f) = settings::fields().into_iter().find(|(_, f)| f.key == "timeline.dropImportToTimeline").expect("registry field"); + assert_eq!(cat, "timeline"); + assert!(matches!(f.kind, Kind::Bool)); + assert_eq!(settings::field("timeline.dropImportToTimeline").map(|f| f.key), Some("timeline.dropImportToTimeline")); +} + /// New users keep the dark Darkest theme (following the system is opt-in); a file saved before /// appearance modes keeps showing its Color Theme as a fixed Dark or Light mode. #[test] diff --git a/crates/engine/src/shortcuts.rs b/crates/engine/src/shortcuts.rs index 52888624c..0c9e47718 100644 --- a/crates/engine/src/shortcuts.rs +++ b/crates/engine/src/shortcuts.rs @@ -426,6 +426,7 @@ fn category_of(id: &str, menu: &[&str]) -> String { "view" => "View".into(), "app" | "help" | "mode" => "Application".into(), "captions" => "Captions".into(), + "tts" => "Text to Speech".into(), _ => menu.first().map(|s| s.to_string()).unwrap_or_else(|| { let mut c = prefix.chars(); c.next().map(|f| f.to_uppercase().collect::() + c.as_str()).unwrap_or_default() diff --git a/crates/engine/src/source_monitor.rs b/crates/engine/src/source_monitor.rs index 836fff49e..cdbcf35f7 100644 --- a/crates/engine/src/source_monitor.rs +++ b/crates/engine/src/source_monitor.rs @@ -94,7 +94,7 @@ fn marker_list(p: &mut Project, item: ItemId) -> Result<&mut Vec> { let it = p.item_mut(item).ok_or_else(|| EngineError::Other("Source item is unavailable".into()))?; match &mut it.kind { filmcraft_project::ItemKind::Media(m) => Ok(&mut m.markers), - filmcraft_project::ItemKind::Sequence(q) => Ok(&mut q.markers), + filmcraft_project::ItemKind::Sequence(q) => Ok(&mut std::sync::Arc::make_mut(q).markers), _ => Err(EngineError::Other("Source marker editing requires media or a sequence".into())), } } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 15af29270..9852734d7 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -326,6 +326,92 @@ fn add_keyframe_toggles_the_keyframe_at_the_playhead() { assert_eq!(scale(&s), (0, 100.0)); } +/// `effects.setParam` writes a keyframe only on an animated parameter. An agent that sends a value +/// with a `time` to a parameter that is not animated gets a static value (the result says so: +/// 0 keyframes); `"keyframe": true` is the stopwatch and the value in one step. +#[test] +fn set_param_writes_a_keyframe_when_asked_or_animated() { + let mut s = demo(); + let c = s.active_sequence().unwrap().video_tracks[0].items[0].id.0; + // (keyframes, static value, value at 0.5 s, value at 2.5 s) + let scale = |s: &Session| { + let it = s.active_sequence().unwrap().find_item(filmcraft_project::ClipId(c)).unwrap().1; + let p = &it.effect("motion").unwrap().params["scale"]; + let at = |seconds: f64| p.f64_at(it.source_time_at(Tick::from_seconds_f64(seconds))); + (p.keyframes.len(), p.value.as_f64().unwrap(), at(0.5), at(2.5)) + }; + let set = |value: f64, seconds: f64| json!({"clip": c, "effect": "motion", "param": "scale", "value": value, "seconds": seconds}); + // not animated: the time is not used and the value is static + let r = s.execute("effects.setParam", set(80.0, 0.5)).unwrap(); + assert_eq!(r["keyframes"], 0, "{r}"); + assert_eq!(scale(&s), (0, 80.0, 80.0, 80.0)); + // `keyframe`: the parameter becomes animated at that time + let mut first = set(60.0, 0.5); + first["keyframe"] = json!(true); + let r = s.execute("effects.setParam", first).unwrap(); + assert_eq!(r["keyframes"], 1, "{r}"); + assert_eq!(scale(&s).0, 1); + // animated now, so a plain set at another time is a second keyframe + let r = s.execute("effects.setParam", set(120.0, 2.5)).unwrap(); + assert_eq!(r["keyframes"], 2, "{r}"); + let (n, _, a, b) = scale(&s); + assert_eq!((n, a, b), (2, 60.0, 120.0)); + // a keyframe that is already there is replaced, not doubled + let mut again = set(70.0, 0.5); + again["keyframe"] = json!(true); + assert_eq!(s.execute("effects.setParam", again).unwrap()["keyframes"], 2); + assert_eq!(scale(&s).2, 70.0); + s.undo(); + assert_eq!(scale(&s).2, 60.0); + s.undo(); + assert_eq!(scale(&s).0, 1); + s.redo(); + s.redo(); + let (n, _, a, b) = scale(&s); + assert_eq!((n, a, b), (2, 70.0, 120.0)); + // a parameter that cannot be animated refuses the keyframe and keeps its value + let blend = json!({"clip": c, "effect": "opacity", "param": "blend", "value": 8, "keyframe": true}); + let e = s.execute("effects.setParam", blend).unwrap_err().to_string(); + assert!(e.contains("cannot be animated"), "{e}"); + let it = s.active_sequence().unwrap().find_item(filmcraft_project::ClipId(c)).unwrap().1; + assert_eq!(it.effect("opacity").unwrap().params["blend"].value, filmcraft_project::ParamValue::Choice(0)); + // `keyframe` that is not a bool means no keyframe request + let mut odd = set(70.0, 0.5); + odd["keyframe"] = json!("yes"); + assert_eq!(s.execute("effects.setParam", odd).unwrap()["keyframes"], 2); +} + +/// A choice parameter takes its option name as well as its index, as transition parameters do +/// (`effects.list {"detail": true}` lists the names). +#[test] +fn set_param_takes_a_choice_by_name() { + use filmcraft_project::ParamValue; + let mut s = demo(); + let c = s.active_sequence().unwrap().video_tracks[0].items[0].id.0; + let blend = + |s: &Session| s.active_sequence().unwrap().find_item(filmcraft_project::ClipId(c)).unwrap().1.effect("opacity").unwrap().params["blend"].value.clone(); + let set = |value: serde_json::Value| json!({"clip": c, "effect": "opacity", "param": "blend", "value": value}); + s.execute("effects.setParam", set(json!("Screen"))).unwrap(); + assert_eq!(blend(&s), ParamValue::Choice(8)); + s.execute("effects.setParam", set(json!("linear dodge (add)"))).unwrap(); + assert_eq!(blend(&s), ParamValue::Choice(10), "names are matched without case"); + s.execute("effects.setParam", set(json!(12))).unwrap(); + assert_eq!(blend(&s), ParamValue::Choice(12), "an index works as before"); + s.undo(); + assert_eq!(blend(&s), ParamValue::Choice(10)); + s.redo(); + assert_eq!(blend(&s), ParamValue::Choice(12)); + // a name that is not an option is an error that lists the options and changes nothing + let e = s.execute("effects.setParam", set(json!("Glow"))).unwrap_err().to_string(); + assert!(e.contains("Screen") && e.contains("Overlay"), "{e}"); + for bad in [json!(""), json!(null), json!([8]), json!({"name": "Screen"}), json!(-1), json!(1.5)] { + assert!(s.execute("effects.setParam", set(bad.clone())).is_err(), "{bad}"); + } + assert_eq!(blend(&s), ParamValue::Choice(12)); + // a name is still the wrong type for a parameter that is not a choice + assert!(s.execute("effects.setParam", json!({"clip": c, "effect": "motion", "param": "scale", "value": "Screen"})).is_err()); +} + /// Dragging the Volume line or one of its keyframes in the timeline (#223) sends a keyframe edit /// every frame; `merge` keeps the whole drag one undo step and `begin` starts the next one. #[test] @@ -731,6 +817,114 @@ fn dragging_an_effect_parameter_is_one_undo_step() { assert_eq!(opacity(&s), start, "and then the whole first one"); } +/// The demo's first video transition between two clips: (id, start, duration, cut). +fn demo_crossing(s: &Session) -> (u64, Tick, Tick, Tick) { + let q = s.active_sequence().unwrap(); + q.video_tracks + .iter() + .find_map(|t| { + t.transitions.iter().find(|x| x.from.is_some() && x.to.is_some()).map(|x| (x.id.0, x.start, x.duration, t.item(x.to.unwrap()).unwrap().start)) + }) + .expect("the demo has a transition between two clips") +} + +fn transition_ids(s: &Session) -> Vec { + s.active_sequence().unwrap().all_tracks().flat_map(|t| t.transitions.iter().map(|x| x.id.0)).collect() +} + +/// #430: a transition can be selected (it replaces the clip selection and back), Delete removes +/// it and leaves its clips, undo brings it back; Ripple Delete is not for a transition alone. +#[test] +fn transitions_select_and_delete() { + let mut s = demo(); + let (tid, ..) = demo_crossing(&s); + let clips = s.active_sequence().unwrap().all_tracks().map(|t| t.items.len()).sum::(); + let some_clip = s.active_sequence().unwrap().video_tracks[0].items[0].id.0; + s.execute("timeline.select", json!({"clips": [some_clip]})).unwrap(); + let r = s.execute("timeline.select", json!({"transitions": [tid]})).unwrap(); + assert_eq!(r["transitionSelection"], json!([tid])); + assert!(s.state.selection.is_empty(), "selecting a transition replaces the clip selection"); + assert!(s.execute("timeline.select", json!({"transitions": [987_654]})).is_err(), "unknown ids are refused"); + assert!(s.execute("edit.rippleDelete", json!({})).is_err(), "Ripple Delete needs clips"); + let before = (*s.project).clone(); + s.execute("edit.clear", json!({})).unwrap(); + assert!(!transition_ids(&s).contains(&tid), "Delete removes the selected transition"); + assert_eq!(s.active_sequence().unwrap().all_tracks().map(|t| t.items.len()).sum::(), clips, "and leaves the clips"); + assert!(s.state.transition_selection.is_empty()); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(*s.project, before, "one undo step"); + // selecting clips clears a transition selection + s.execute("timeline.select", json!({"transitions": [tid]})).unwrap(); + s.execute("timeline.select", json!({"clips": [some_clip]})).unwrap(); + assert!(s.state.transition_selection.is_empty()); + // by id, without a selection + s.execute("sequence.removeTransition", json!({"transitions": [tid]})).unwrap(); + assert!(!transition_ids(&s).contains(&tid)); + assert!(s.execute("sequence.removeTransition", json!({"transitions": [tid]})).is_err(), "nothing left to remove"); + assert!(s.execute("sequence.removeTransition", json!({})).is_err(), "nothing given or selected"); +} + +/// #224 / Effect Controls: Duration follows the alignment, Alignment re-aligns on the cut, Start +/// moves it (Custom Start), all within the clips it joins; a drag is one undo step. +#[test] +fn transition_duration_alignment_and_position() { + let mut s = demo(); + let rate = s.sequence_rate(); + let f = |n: i64| rate.tick_of(n); + let (tid, _, _, cut) = demo_crossing(&s); + let set = |s: &mut Session, p: Value| { + let mut p = p; + p["transition"] = json!(tid); + s.execute("sequence.setTransition", p) + }; + let r = set(&mut s, json!({"align": "start"})).unwrap(); + assert_eq!((r["start"].as_i64(), r["align"].as_str()), (Some(cut.0), Some("start"))); + // Start at Cut: only the end moves + let r = set(&mut s, json!({"frames": 10})).unwrap(); + assert_eq!((r["start"].as_i64(), r["duration"].as_i64()), (Some(cut.0), Some(f(10).0))); + // End at Cut: only the beginning moves + let r = set(&mut s, json!({"align": "end", "frames": 8})).unwrap(); + assert_eq!((r["start"].as_i64(), r["duration"].as_i64(), r["align"].as_str()), (Some((cut - f(8)).0), Some(f(8).0), Some("end"))); + // Center at Cut: both ends + let r = set(&mut s, json!({"align": "center", "frames": 12})).unwrap(); + assert_eq!((r["start"].as_i64(), r["align"].as_str()), (Some((cut - f(6)).0), Some("center"))); + // dragging it over the cut: Custom Start; then a new duration keeps its middle + let r = set(&mut s, json!({"start": (cut - f(3)).0})).unwrap(); + assert_eq!(r["align"], json!("custom")); + let r = set(&mut s, json!({"frames": 6})).unwrap(); + assert_eq!((r["start"].as_i64(), r["duration"].as_i64()), (Some(cut.0), Some(f(6).0)), "centre at cut + 3 frames, 6 frames long"); + // dragging one edge: start and duration together + let r = set(&mut s, json!({"start": (cut - f(4)).0, "duration": f(10).0})).unwrap(); + assert_eq!((r["start"].as_i64(), r["duration"].as_i64()), (Some((cut - f(4)).0), Some(f(10).0))); + // refused, and nothing changes + let before = (*s.project).clone(); + let history = s.history.undo.len(); + for p in [ + json!({"frames": 0}), + json!({"frames": 100_000_000}), + json!({"duration": -5}), + json!({"duration": i64::MAX}), + json!({"frames": 10, "duration": f(10).0}), + json!({"align": "middle"}), + json!({"align": "start", "start": cut.0}), + json!({"start": i64::MIN}), + json!({"start": (cut + f(1)).0, "duration": f(1).0}), + ] { + assert!(set(&mut s, p.clone()).is_err(), "{p}"); + assert_eq!(*s.project, before, "{p}"); + assert_eq!(s.history.undo.len(), history, "{p}"); + } + assert!(s.execute("sequence.setTransition", json!({"transition": 987_654, "frames": 5})).is_err()); + // a drag (merge) is one undo step + let history = s.history.undo.len(); + for (i, n) in [11, 12, 13, 14].into_iter().enumerate() { + set(&mut s, json!({"frames": n, "merge": true, "begin": i == 0})).unwrap(); + } + assert_eq!(s.history.undo.len(), history + 1); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(*s.project, before); +} + /// #484: Enable flips each selected clip on its own, as in Premiere. With one enabled and one /// disabled clip selected it swaps them, instead of first making both the same. #[test] diff --git a/crates/engine/src/voiceover.rs b/crates/engine/src/voiceover.rs index 6811580f8..fa3dd42a7 100644 --- a/crates/engine/src/voiceover.rs +++ b/crates/engine/src/voiceover.rs @@ -386,6 +386,13 @@ fn sync(s: &mut Session, p: &Value) -> Result { /// Directory recordings are written to. fn record_dir(s: &Session, p: &Value) -> String { + media_dir(s, p, "Voice-over Recordings") +} + +/// Directory generated media (recordings, narrations) is written to: `dir`, else Project +/// Settings ▸ Scratch Disks ▸ Captured, else next to the project, else `fallback` in the data or +/// temporary directory. +pub(crate) fn media_dir(s: &Session, p: &Value, fallback: &str) -> String { if let Some(d) = str_p(p, "dir") { return d.to_string(); } @@ -396,7 +403,21 @@ fn record_dir(s: &Session, p: &Value) -> String { return d.to_string_lossy().into_owned(); } let base = s.prefs_path.as_ref().and_then(|p| p.parent()).map(|d| d.to_path_buf()).unwrap_or_else(|| crate::temp_dir().join("FilmCraft")); - base.join("Voice-over Recordings").to_string_lossy().into_owned() + base.join(fallback).to_string_lossy().into_owned() +} + +/// `/ .wav` with the first `n` used by neither a project item nor an existing file; +/// an error once `n` passes 99 999 (bounded: `file_size` is a filesystem call per try). +pub(crate) fn unique_wav_path(s: &Session, dir: &str, base: &str) -> Result { + let names: std::collections::HashSet = s.project.items.values().map(|i| i.name.clone()).collect(); + for k in 1..=99_999u32 { + let file = format!("{base} {k}.wav"); + let path = std::path::Path::new(dir).join(&file).to_string_lossy().into_owned(); + if !names.contains(&file) && s.services.file_size(&path).is_err() { + return Ok(path); + } + } + Err(EngineError::Other(format!("{dir}: no free file name for \"{base} .wav\""))) } fn stop(s: &mut Session, p: &Value) -> Result { @@ -449,16 +470,7 @@ fn stop(s: &mut Session, p: &Value) -> Result { // write the file let dir = record_dir(s, p); let base = s.prefs.voice_over.name.clone(); - let names: std::collections::HashSet = s.project.items.values().map(|i| i.name.clone()).collect(); - let mut k = 1; - let path = loop { - let file = format!("{base} {k}.wav"); - let path = std::path::Path::new(&dir).join(&file).to_string_lossy().into_owned(); - if !names.contains(&file) && s.services.file_size(&path).is_err() { - break path; - } - k += 1; - }; + let path = unique_wav_path(s, &dir, &base)?; if !cfg!(target_arch = "wasm32") { std::fs::create_dir_all(&dir).map_err(|e| EngineError::Other(format!("{dir}: {e}")))?; } diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index 1abdc6668..238649afd 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -35,6 +35,8 @@ //! (`Project::source_graphics`). No-op step; older builds would drop these fields when saving. //! - **v13**: all container audio streams, and per-timeline-clip stream selection. Old single-stream //! `audio` objects read as stream zero; older builds must refuse new multi-stream projects. +//! - **v14** (M7.9): narrations (`Project::narrations`, Text to Speech). No-op step; older builds +//! would drop them when saving. #![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] @@ -56,7 +58,7 @@ pub type Migration = fn(Value) -> Result; /// `MIGRATIONS[i]` upgrades schema `i + 1` to `i + 2`. Append one function per schema bump; never /// edit a shipped one. pub const MIGRATIONS: &[Migration] = - &[v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12, v12_to_v13]; + &[v1_to_v2, v2_to_v3, v3_to_v4, v4_to_v5, v5_to_v6, v6_to_v7, v7_to_v8, v8_to_v9, v9_to_v10, v10_to_v11, v11_to_v12, v12_to_v13, v13_to_v14]; /// The schema version this build writes (and the newest it reads). pub const SCHEMA_VERSION: u32 = MIGRATIONS.len() as u32 + 1; @@ -224,6 +226,7 @@ fn validate_loaded(mut loaded: Loaded) -> Result { m.info.audio_streams.truncate(max); } filmcraft_project::ItemKind::Sequence(sequence) => { + let sequence = std::sync::Arc::make_mut(sequence); for clip in sequence.audio_tracks.iter_mut().flat_map(|t| t.items.iter_mut()) { if clip.audio_stream >= max { log::warn!("sequence `{name}`: clip `{}` uses audio stream {}, reset to 0", clip.name, clip.audio_stream); @@ -346,13 +349,18 @@ fn v12_to_v13(doc: Value) -> Result { Ok(doc) } +/// v13 → v14: narrations (Text to Speech). Existing data needs no change. +fn v13_to_v14(doc: Value) -> Result { + Ok(doc) +} + #[cfg(test)] mod tests { use super::*; #[test] fn schema_version_matches_table() { - assert_eq!(SCHEMA_VERSION, 13); + assert_eq!(SCHEMA_VERSION, 14); } #[test] diff --git a/crates/format/tests/fixtures.rs b/crates/format/tests/fixtures.rs index 7cfe2281e..e99a7853f 100644 --- a/crates/format/tests/fixtures.rs +++ b/crates/format/tests/fixtures.rs @@ -131,7 +131,7 @@ pub fn big_project(n: usize) -> Project { let seq_id = ItemId(21); let mut next = 1_000_000u64; let seq = match &mut p.items.get_mut(&seq_id).unwrap().kind { - ItemKind::Sequence(s) => s, + ItemKind::Sequence(s) => std::sync::Arc::make_mut(s), _ => unreachable!(), }; let v0 = seq.video_tracks[0].items[0].clone(); diff --git a/crates/interchange/src/common.rs b/crates/interchange/src/common.rs index 45dfa500f..2c701881c 100644 --- a/crates/interchange/src/common.rs +++ b/crates/interchange/src/common.rs @@ -387,7 +387,7 @@ impl Builder { /// Add an empty sequence item (filled later with [`Builder::put_sequence`]). pub fn reserve_sequence(&mut self, name: &str, settings: SequenceSettings, bin: Option) -> ItemId { let seq = empty_sequence(settings); - self.p.add_item(name, Label::Forest, ItemKind::Sequence(Box::new(seq)), bin) + self.p.add_item(name, Label::Forest, ItemKind::Sequence(std::sync::Arc::new(seq)), bin) } pub fn put_sequence(&mut self, id: ItemId, mut seq: Sequence) { diff --git a/crates/interchange/src/fcp7.rs b/crates/interchange/src/fcp7.rs index 1b94f7122..d5de669d2 100644 --- a/crates/interchange/src/fcp7.rs +++ b/crates/interchange/src/fcp7.rs @@ -110,6 +110,24 @@ struct Imp<'a, 'i, 'r> { in_progress: HashSet, links: Vec, xml_clip_ids: HashMap<(ItemId, String), ClipId>, + /// `(sequence key, file key)` pairs where some audio clip item of that file uses `sourcetrack/trackindex` > 1, + /// i.e. the file's channels are split onto mono tracks in that sequence (#463). + split_files: HashSet<(String, String)>, +} + +/// The `(sequence key, file key)` an audio clip item belongs to, if it references a file. +fn clip_seq_file(c: Node) -> Option<(String, String)> { + let seq = c.ancestors().skip(1).find(|a| a.has_tag_name("sequence"))?; + Some((node_key(seq), node_key(child(c, "file")?))) +} + +/// The 1-based `sourcetrack/trackindex` of an audio clip item. +fn audio_track_index(c: Node) -> Option { + let st = child(c, "sourcetrack")?; + if child_text(st, "mediatype").is_some_and(|m| m != "audio") { + return None; + } + child_i64(st, "trackindex") } fn node_key(n: Node) -> String { @@ -136,9 +154,15 @@ pub(crate) fn import(text_in: &str, opts: &ImportOptions, report: &mut Report) - in_progress: HashSet::new(), links: Vec::new(), xml_clip_ids: HashMap::new(), + split_files: HashSet::new(), }; for n in root.descendants().filter(|n| n.is_element()) { match n.tag_name().name() { + "clipitem" if audio_track_index(n).is_some_and(|i| i > 1) => { + if let Some(key) = clip_seq_file(n) { + imp.split_files.insert(key); + } + } "file" if elements(n).next().is_some() => { imp.files.entry(node_key(n)).or_insert(n); } @@ -482,6 +506,17 @@ impl<'a, 'i> Imp<'a, 'i, '_> { if let Some(l) = path_text(c, &["labels", "label2"]).and_then(label_from) { ti.label = l; } + // `sourcetrack/trackindex` (1-based) picks the source channel of a mono track split off a multichannel file (#463). + // Index 1 is also what a stereo clip on a stereo track carries, so it means channel 0 only when another clip + // item of the same file in this sequence uses a higher index (the file is split); otherwise it keeps the + // default (empty) channel mapping. + if kind == TrackKind::Audio + && let Some(idx) = audio_track_index(c) + && (idx > 1 || (idx == 1 && clip_seq_file(c).is_some_and(|k| self.split_files.contains(&k)))) + && let Ok(ch) = u16::try_from(idx - 1) + { + ti.source_channels = vec![ch]; + } if let Some(mode) = child_text(c, "compositemode") { match BLEND_XML.iter().find(|(n, _)| n.eq_ignore_ascii_case(mode)) { Some((_, v)) if *v != 0 => set_param(&mut ti, "opacity", "blend", Param::new(ParamValue::Choice(*v))), @@ -962,7 +997,7 @@ impl Exp<'_, '_> { if kind == TrackKind::Audio { self.w.open("sourcetrack", &[]); self.w.text("mediatype", "audio"); - self.w.text("trackindex", 1); + self.w.text("trackindex", c.source_channels.first().map_or(1, |ch| u32::from(*ch).saturating_add(1))); self.w.close(); } if let Some(blend) = param(c, "opacity", "blend").and_then(|p| p.value.as_f64()).map(|v| v as u32).filter(|v| *v != 0) { diff --git a/crates/interchange/src/lib.rs b/crates/interchange/src/lib.rs index 61f46275a..ad0537551 100644 --- a/crates/interchange/src/lib.rs +++ b/crates/interchange/src/lib.rs @@ -438,6 +438,7 @@ pub fn merge_into(target: &mut Project, fragment: Imported, bin: Option) it.created = it.id.0; match &mut it.kind { ItemKind::Sequence(seq) => { + let seq = std::sync::Arc::make_mut(seq); for m in &mut seq.markers { m.id = MarkerId(target.alloc_id()); } @@ -510,6 +511,7 @@ pub fn rebase_source_timecode(project: &mut Project, item: ItemId, start_tc_fram } for it in project.items.values_mut() { if let ItemKind::Sequence(seq) = &mut it.kind { + let seq = std::sync::Arc::make_mut(seq); for t in seq.all_tracks_mut() { for ti in t.items.iter_mut().filter(|ti| ti.item == item) { ti.source_in -= off; diff --git a/crates/interchange/tests/fcp7.rs b/crates/interchange/tests/fcp7.rs index 4d3677efc..6b2a9414b 100644 --- a/crates/interchange/tests/fcp7.rs +++ b/crates/interchange/tests/fcp7.rs @@ -527,3 +527,76 @@ fn sub_frame_clips_keep_equal_source_and_record_spans() { assert_eq!(out_f - in_f, end_f - start_f, "start {start}, in {src}, duration {dur} (tenths of a second): {spans:?}"); } } + +/// #463: a stereo file split onto two mono tracks (A1 = channel 1, A2 = channel 2) lost the right channel, +/// because every audio clip item was written with `sourcetrack/trackindex` 1 and the importer ignored it. +#[test] +fn split_mono_tracks_keep_their_source_channel() { + let r = FrameRate::FPS_23_976; + let mut p = Project::new("P"); + let a = media(&mut p, "/m/a.wav", false, true, r); + let s = sequence(&mut p, "t", r, false); + let left = clip(&mut p, s, TrackKind::Audio, 0, a, 0, 24, 0); + let right = clip(&mut p, s, TrackKind::Audio, 1, a, 0, 24, 0); + for (id, ch) in [(left, 0u16), (right, 1u16)] { + let (_, it) = p.sequence_mut(s).unwrap().find_item_mut(id).unwrap(); + it.source_channels = vec![ch]; + } + let (imp, xml, _) = roundtrip(&p, s, Format::Fcp7Xml, &ExportOptions::default()); + assert!(xml.contains("2"), "{xml}"); + let seq = imp.project.sequence(only_seq(&imp)).unwrap(); + let channels = |t: usize| seq.tracks(TrackKind::Audio)[t].items.iter().map(|i| i.source_channels.clone()).collect::>(); + assert_eq!(channels(0), vec![vec![0u16]], "{xml}"); + assert_eq!(channels(1), vec![vec![1u16]], "{xml}"); + let (bytes, _) = export(&imp.project, only_seq(&imp), Format::Fcp7Xml, &ExportOptions::default()).expect("export"); + assert!(String::from_utf8(bytes).unwrap().contains("2")); +} + +#[test] +fn audio_sourcetrack_trackindex_2_imports_as_source_channel_1() { + let doc = format!( + r#" + +S48{RATE} + +"# + ); + let (imp, _) = import(doc.as_bytes(), Format::Fcp7Xml, None).unwrap(); + let seq = imp.project.sequence(only_seq(&imp)).unwrap(); + assert_eq!(seq.tracks(TrackKind::Audio)[0].items[0].source_channels, vec![1u16]); +} + +/// #463: Premiere writes a stereo file split onto two mono tracks as clip items with `trackindex` 1 and 2; +/// a file used only with `trackindex` 1 is a stereo clip and keeps the default (all channels) mapping. +#[test] +fn audio_trackindex_1_is_channel_0_only_when_the_file_is_split() { + let doc = format!( + r#" + +S48{RATE} + +"# + ); + let (imp, _) = import(doc.as_bytes(), Format::Fcp7Xml, None).unwrap(); + let seq = imp.project.sequence(only_seq(&imp)).unwrap(); + let a1 = &seq.tracks(TrackKind::Audio)[0].items; + assert_eq!(a1[0].source_channels, vec![0u16]); + assert!(a1[1].source_channels.is_empty(), "{:?}", a1[1].source_channels); + assert_eq!(seq.tracks(TrackKind::Audio)[1].items[0].source_channels, vec![1u16]); +} diff --git a/crates/platform/Cargo.toml b/crates/platform/Cargo.toml index 02e5c306a..9e0758066 100644 --- a/crates/platform/Cargo.toml +++ b/crates/platform/Cargo.toml @@ -50,6 +50,7 @@ windows = { version = "0.62", features = [ "Win32_Media_MediaFoundation", "Win32_System_Com", "Win32_System_LibraryLoader", + "Win32_UI_WindowsAndMessaging", ] } # Linux: libva (VA-API) is loaded at run time with libloading (ISC; already in the lockfile through diff --git a/crates/platform/src/cursor.rs b/crates/platform/src/cursor.rs new file mode 100644 index 000000000..847441699 --- /dev/null +++ b/crates/platform/src/cursor.rs @@ -0,0 +1,28 @@ +//! The mouse position in screen pixels, for when the windowing layer has none. +//! +//! While files are dragged in from the OS file manager, winit on Windows delivers no cursor events +//! (it only reports hovered and dropped files), so the UI cannot learn where a drop landed. +//! +//! This is the one non-media OS call in this crate: `crates/platform` is the only crate allowed +//! `unsafe` (AGENTS.md §0.3, ADR 0001), so the single `GetCursorPos` call lives here rather than +//! in the UI. It is read-only, has a safe `Option`-returning API, and every other platform (and a +//! failed call) falls back to the last pointer position egui saw. Drop it once winit reports the +//! pointer position during an OS drag-and-drop. + +/// The cursor's position on the virtual screen in physical pixels, or `None` where the OS cannot +/// say or this platform's windowing layer already reports it during a drag. +#[cfg(target_os = "windows")] +#[allow(unsafe_code)] +pub fn cursor_screen_position() -> Option<(i32, i32)> { + use windows::Win32::Foundation::POINT; + use windows::Win32::UI::WindowsAndMessaging::GetCursorPos; + let mut p = POINT { x: 0, y: 0 }; + // SAFETY: `p` is a valid, writable POINT for the duration of the call. + unsafe { GetCursorPos(&mut p) }.ok()?; + Some((p.x, p.y)) +} + +#[cfg(not(target_os = "windows"))] +pub fn cursor_screen_position() -> Option<(i32, i32)> { + None +} diff --git a/crates/platform/src/lib.rs b/crates/platform/src/lib.rs index 2ddf5fabb..8744da0eb 100644 --- a/crates/platform/src/lib.rs +++ b/crates/platform/src/lib.rs @@ -34,6 +34,7 @@ mod annexb; #[cfg_attr(not(any(target_os = "windows", target_os = "linux")), allow(dead_code))] mod biplanar; +pub mod cursor; #[cfg(target_os = "macos")] pub mod hardware_encode; pub mod hybrid; diff --git a/crates/platform/src/nvenc/abi_tests.rs b/crates/platform/src/nvenc/abi_tests.rs index 270108a58..3c66823e9 100644 --- a/crates/platform/src/nvenc/abi_tests.rs +++ b/crates/platform/src/nvenc/abi_tests.rs @@ -126,6 +126,25 @@ fn layouts_match_the_header() { (112, 4), "NV_ENC_CONFIG_HEVC_VUI_PARAMETERS" ); + // The header typedefs the HEVC VUI to the H.264 one; pin the fields session.rs writes, both + // inside the VUI and where they land in NV_ENC_CONFIG_HEVC (hevcVUIParameters at 64), so a + // binding mismatch cannot hide behind the typedef (#442: HEVC timing came out reversed). + for (field, offset, name) in [ + (offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, videoSignalTypePresentFlag), 8, "videoSignalTypePresentFlag"), + (offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, videoFormat), 12, "videoFormat"), + (offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, videoFullRangeFlag), 16, "videoFullRangeFlag"), + (offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, colourDescriptionPresentFlag), 20, "colourDescriptionPresentFlag"), + (offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, colourPrimaries), 24, "colourPrimaries"), + (offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, transferCharacteristics), 28, "transferCharacteristics"), + (offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, colourMatrix), 32, "colourMatrix"), + (offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, timingInfoPresentFlag), 52, "timingInfoPresentFlag"), + (offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, numUnitInTicks), 56, "numUnitInTicks"), + (offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, timeScale), 60, "timeScale"), + (offset_of!(NV_ENC_CONFIG_HEVC_VUI_PARAMETERS, reserved), 64, "reserved"), + ] { + assert_eq!(field, offset, "NV_ENC_CONFIG_HEVC_VUI_PARAMETERS.{name}"); + assert_eq!(offset_of!(NV_ENC_CONFIG_HEVC, hevcVUIParameters) + field, 64 + offset, "NV_ENC_CONFIG_HEVC.hevcVUIParameters.{name}"); + } assert_eq!((size_of::(), align_of::()), (1560, 8), "NV_ENC_CONFIG_HEVC"); assert_eq!(offset_of!(NV_ENC_CONFIG_HEVC, level), 0, "NV_ENC_CONFIG_HEVC.level"); assert_eq!(offset_of!(NV_ENC_CONFIG_HEVC, tier), 4, "NV_ENC_CONFIG_HEVC.tier"); diff --git a/crates/platform/src/nvenc/session.rs b/crates/platform/src/nvenc/session.rs index b19691f71..8395f6824 100644 --- a/crates/platform/src/nvenc/session.rs +++ b/crates/platform/src/nvenc/session.rs @@ -416,9 +416,11 @@ impl Session { vui.colourPrimaries = u32::from(p.signal.primaries); vui.transferCharacteristics = u32::from(p.signal.transfer); vui.colourMatrix = u32::from(p.signal.matrix); - vui.timingInfoPresentFlag = 1; - vui.numUnitInTicks = p.fps.1; - vui.timeScale = p.fps.0; + // Let NVENC derive timing from frameRateNum/frameRateDen. Some drivers + // reverse the explicitly supplied HEVC VUI timing pair. + vui.timingInfoPresentFlag = 0; + vui.numUnitInTicks = 0; + vui.timeScale = 0; } } diff --git a/crates/platform/tests/nvenc_hevc.rs b/crates/platform/tests/nvenc_hevc.rs index d10835298..acd2bee9b 100644 --- a/crates/platform/tests/nvenc_hevc.rs +++ b/crates/platform/tests/nvenc_hevc.rs @@ -321,3 +321,19 @@ fn a_keyframe_every_one_or_two_pictures_is_written_without_b_frames() { assert_eq!(n + dec.flush().len(), 12, "keyint {keyint}"); } } + +#[test] +fn vui_timing_matches_integer_and_fractional_frame_rates() { + let _one = SERIAL.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for fps in [(24, 1), (25, 1), (30_000, 1_001)] { + let mut cfg = config(640, 360); + cfg.fps = fps; + cfg.bframes = false; + let Some((enc, _, _)) = encode_all(&cfg, 2) else { return }; + let sps_nal = enc.parameter_sets().0; + let sps = Sps::parse(&unescape_rbsp(&sps_nal[2..])).unwrap(); + let (units, scale) = sps.vui.and_then(|v| v.timing).expect("HEVC VUI timing"); + assert!(units > 0 && scale > 0); + assert_eq!(u64::from(scale) * u64::from(fps.1), u64::from(units) * u64::from(fps.0), "timing for {fps:?}"); + } +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index 9d67b5c8c..e8a988568 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -20,6 +20,7 @@ pub mod keyframe; pub mod mask; pub mod mixer; pub mod multicam; +pub mod narration; pub mod transcript; pub mod vtransition; @@ -39,6 +40,7 @@ pub use keyframe::{Interpolation, Keyframe, Param, ParamValue}; pub use mask::{Mask, MaskMode, MaskPath, MaskVertex, TrackMethod}; pub use mixer::{AutomationMode, InputMap, MixerStrip, TrackSend}; pub use multicam::{Camera, MergedClip, MulticamAudio, MulticamSel, MulticamSource}; +pub use narration::{Narration, VocalPitch}; pub use transcript::{Speaker, Transcript, Word}; macro_rules! id_type { @@ -262,7 +264,9 @@ impl MediaClip { #[allow(clippy::large_enum_variant)] pub enum ItemKind { Media(MediaClip), - Sequence(Box), + /// Shared so project snapshots used by undo and background readers do not deep-copy every + /// timeline. Mutators detach with `Arc::make_mut`. + Sequence(std::sync::Arc), /// A subclip: a media item restricted to a range. Subclip { parent: ItemId, @@ -327,7 +331,7 @@ impl ProjectItem { } pub fn as_sequence_mut(&mut self) -> Option<&mut Sequence> { match &mut self.kind { - ItemKind::Sequence(s) => Some(s), + ItemKind::Sequence(s) => Some(std::sync::Arc::make_mut(s)), _ => None, } } @@ -1245,6 +1249,10 @@ pub struct Project { /// project item. Schema v12. #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] pub source_graphics: BTreeMap, + /// Narrations (Text to Speech): the script and voice settings each generated WAV item was + /// made from, keyed by that item. Schema v14. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub narrations: BTreeMap, } /// How a sequence is shown in the Timeline panel: its zoom, scroll position and track heights. @@ -1339,6 +1347,7 @@ impl Project { transcripts: BTreeMap::new(), search_bins: Vec::new(), source_graphics: BTreeMap::new(), + narrations: BTreeMap::new(), } } @@ -1502,7 +1511,7 @@ impl Project { let id = TrackId(self.alloc_id()); seq.audio_tracks.push(Track::new(id, TrackKind::Audio, format!("Audio {}", i + 1))); } - self.add_item(name, Label::Forest, ItemKind::Sequence(Box::new(seq)), bin) + self.add_item(name, Label::Forest, ItemKind::Sequence(std::sync::Arc::new(seq)), bin) } /// Re-base an item's media time: whatever was at media time `t` is at `t - delta` afterwards @@ -1524,6 +1533,7 @@ impl Project { } for it in self.items.values_mut() { if let ItemKind::Sequence(seq) = &mut it.kind { + let seq = std::sync::Arc::make_mut(seq); for t in seq.all_tracks_mut() { for ti in t.items.iter_mut().filter(|ti| users.contains(&ti.item)) { ti.source_in -= delta; @@ -1984,6 +1994,30 @@ mod tests { assert_eq!(p.root.parent_of(i), None); } + #[test] + fn project_clone_shares_sequences_until_mutated() { + fn sequence_arc(p: &Project, id: ItemId) -> &std::sync::Arc { + match &p.item(id).unwrap().kind { + ItemKind::Sequence(q) => q, + _ => unreachable!(), + } + } + + let (mut p, _, a) = demo_project(); + let b = p.new_sequence("Sequence 02", SequenceSettings::default(), 1, 1, None); + let before = p.clone(); + + assert!(std::sync::Arc::ptr_eq(sequence_arc(&p, a), sequence_arc(&before, a))); + assert!(std::sync::Arc::ptr_eq(sequence_arc(&p, b), sequence_arc(&before, b))); + + p.sequence_mut(a).unwrap().mark_in = Some(Tick(1)); + + assert!(!std::sync::Arc::ptr_eq(sequence_arc(&p, a), sequence_arc(&before, a))); + assert!(std::sync::Arc::ptr_eq(sequence_arc(&p, b), sequence_arc(&before, b))); + assert_eq!(before.sequence(a).unwrap().mark_in, None); + assert_eq!(Project::from_json(&p.to_json()).unwrap(), p); + } + #[test] fn nest_cycle_finds_sequences_that_contain_themselves() { let (mut p, clip, a) = demo_project(); diff --git a/crates/project/src/narration.rs b/crates/project/src/narration.rs new file mode 100644 index 000000000..5ee13e3c6 --- /dev/null +++ b/crates/project/src/narration.rs @@ -0,0 +1,152 @@ +//! Narrations: audio generated from text (the Text to Speech panel). +//! +//! A [`Narration`] belongs to the project item of the WAV file that was generated for it +//! (`Project::narrations`, keyed by the item id), so every clip of that item can be clicked to +//! reopen the script and voice settings. Editing a narration writes a new WAV (a new item) and +//! swaps the clip over to it; the old item and its narration stay, so undo restores them exactly. + +use filmcraft_time::Tick; +use serde::{Deserialize, Serialize}; + +/// Longest script accepted, in bytes. +pub const MAX_TEXT_BYTES: usize = 64 * 1024; +/// Pace limits (speaking rate multiplier). +pub const MIN_PACE: f64 = 0.5; +pub const MAX_PACE: f64 = 2.0; + +/// Text to Speech ▸ Advanced ▸ Vocal pitch. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum VocalPitch { + ExtraLow, + Low, + #[default] + Default, + High, + ExtraHigh, +} + +impl VocalPitch { + pub const ALL: [VocalPitch; 5] = [VocalPitch::ExtraLow, VocalPitch::Low, VocalPitch::Default, VocalPitch::High, VocalPitch::ExtraHigh]; + + pub fn label(self) -> &'static str { + match self { + VocalPitch::ExtraLow => "Extra low", + VocalPitch::Low => "Low", + VocalPitch::Default => "Default", + VocalPitch::High => "High", + VocalPitch::ExtraHigh => "Extra high", + } + } + + /// The command / serde id (`extraLow`, `low`, `default`, `high`, `extraHigh`). + pub fn id(self) -> &'static str { + match self { + VocalPitch::ExtraLow => "extraLow", + VocalPitch::Low => "low", + VocalPitch::Default => "default", + VocalPitch::High => "high", + VocalPitch::ExtraHigh => "extraHigh", + } + } + + pub fn from_id(s: &str) -> Option { + let n = s.to_ascii_lowercase().replace([' ', '_', '-'], ""); + Self::ALL.into_iter().find(|p| p.id().to_ascii_lowercase() == n) + } + + /// Pitch shift in semitones. + pub fn semitones(self) -> f64 { + match self { + VocalPitch::ExtraLow => -6.0, + VocalPitch::Low => -3.0, + VocalPitch::Default => 0.0, + VocalPitch::High => 3.0, + VocalPitch::ExtraHigh => 6.0, + } + } +} + +/// The script and voice settings a narration was generated from. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Narration { + /// The script, with pause markers (`[pause 1s]`) as typed. + pub text: String, + /// BCP 47 language tag (`en-US`). + pub language: String, + /// Voice id (`tts.voices`). + pub voice: String, + #[serde(default)] + pub pitch: VocalPitch, + /// Speaking rate, [`MIN_PACE`]–[`MAX_PACE`]. + pub pace: f64, + /// Length of the speech in the generated file. The file can be longer: when an edit made the + /// speech shorter than its clip, the file is padded with silence to the clip's length. + pub speech_duration: Tick, +} + +impl Narration { + /// Check values read from a project file or a command. + pub fn check(&self) -> Result<(), String> { + if self.text.len() > MAX_TEXT_BYTES { + return Err(format!("the script is longer than {} KB", MAX_TEXT_BYTES / 1024)); + } + if !(MIN_PACE..=MAX_PACE).contains(&self.pace) { + return Err(format!("pace must be between {MIN_PACE} and {MAX_PACE}")); + } + if self.speech_duration < Tick::ZERO { + return Err("negative speech duration".into()); + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> Narration { + Narration { + text: "Hello there. [pause 1s] Welcome.".into(), + language: "en-US".into(), + voice: "basic-female".into(), + pitch: VocalPitch::High, + pace: 1.25, + speech_duration: Tick::from_seconds_f64(2.5), + } + } + + #[test] + fn round_trips_through_json() { + let n = sample(); + let s = serde_json::to_string(&n).unwrap(); + assert!(s.contains("\"pitch\":\"high\""), "{s}"); + assert_eq!(serde_json::from_str::(&s).unwrap(), n); + } + + #[test] + fn pitch_ids_round_trip() { + for p in VocalPitch::ALL { + assert_eq!(VocalPitch::from_id(p.id()), Some(p)); + } + assert_eq!(VocalPitch::from_id("Extra High"), Some(VocalPitch::ExtraHigh)); + assert_eq!(VocalPitch::from_id("loud"), None); + } + + #[test] + fn check_rejects_hostile_values() { + let mut n = sample(); + n.pace = f64::NAN; + assert!(n.check().is_err()); + n.pace = 3.0; + assert!(n.check().is_err()); + let mut n = sample(); + n.text = "a".repeat(MAX_TEXT_BYTES + 1); + assert!(n.check().is_err()); + let mut n = sample(); + n.speech_duration = Tick(-1); + assert!(n.check().is_err()); + assert!(sample().check().is_ok()); + } +} diff --git a/crates/speech/src/models.rs b/crates/speech/src/models.rs index 74fc646c5..4d257159d 100644 --- a/crates/speech/src/models.rs +++ b/crates/speech/src/models.rs @@ -222,15 +222,22 @@ pub type DownloadProgress<'a> = &'a mut dyn FnMut(u64, u64, &str) -> bool; /// verifier. #[cfg(feature = "download")] pub fn download(models_dir: &Path, m: &ModelInfo, progress: DownloadProgress) -> Result<(), crate::SpeechError> { + download_files(&model_dir(models_dir, m), m.files, progress) +} + +/// [`download`] for any pinned file list (the neural voices use it too): missing files of +/// `files` go into `dir`, each verified against its SHA-256 and size before it is renamed into +/// place. +#[cfg(feature = "download")] +pub fn download_files(dir: &Path, files: &[ModelFile], progress: DownloadProgress) -> Result<(), crate::SpeechError> { use crate::SpeechError; use sha2::Digest; use std::io::{Read, Write}; - let dir = model_dir(models_dir, m); - std::fs::create_dir_all(&dir)?; - let total = missing_bytes(models_dir, m); + std::fs::create_dir_all(dir)?; + let total: u64 = files.iter().filter(|f| !std::fs::metadata(dir.join(f.name)).is_ok_and(|md| md.len() == f.size)).map(|f| f.size).sum(); let agent = agent(); let mut done = 0u64; - for f in m.files { + for f in files { let dest = dir.join(f.name); if std::fs::metadata(&dest).is_ok_and(|md| md.len() == f.size) { continue; diff --git a/crates/tts-text/Cargo.toml b/crates/tts-text/Cargo.toml new file mode 100644 index 000000000..4b594930e --- /dev/null +++ b/crates/tts-text/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "filmcraft-tts-text" +description = "FilmCraft text-to-speech text front end: text normalization (numbers, money, dates, times, abbreviations, acronyms, pause markers) into plain spoken words, English and Spanish" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +serde = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/tts-text/README.md b/crates/tts-text/README.md new file mode 100644 index 000000000..d1d385dce --- /dev/null +++ b/crates/tts-text/README.md @@ -0,0 +1,170 @@ +# filmcraft-tts-text + +Text-to-speech text front end for FilmCraft (L2, phases P2a + P2b). Takes a +narration script and returns plain spoken words, structured as tokens: `Word`, +`Pause` (from `[pause 1s]` markers), `SentenceEnd` and `Punct` +(`normalize`), and turns those words into Kokoro phoneme symbols (`phonemize`). + +```rust +use filmcraft_tts_text::{normalize, Lang, Token}; + +let tokens = normalize("Call 555-0134 by 3:30 p.m. on Oct. 7th, it's $3.50 (20% off)", Lang::EnUs)?; +// → Call | five five five zero one three four | by | three thirty p m | on | +// October seventh | , | it's | three dollars and fifty cents | ( | twenty percent off | ) +``` + +English only (`Lang::EnUs`, `Lang::EnGb`), per the amended brief. Handles: +cardinals to 999 999 999 999, negatives, decimals, ordinals, money (`$`, `€`, +`£`), percent, years (1100–2099), clock times, numeric and abbreviated dates, +phone-like digit groups, common abbreviations (Mr., Mrs., Dr., St., Mt., vs., +etc., e.g., i.e., a.m., p.m.), spelled acronyms, `&`/`+`/`@`/`#`/`/`, and pause +markers. Never crashes: >1 MiB input is `NormalizeError::TooLong`, oversized or +leading-zero numbers are read digit by digit, malformed pause markers become +plain text, and all slicing is boundary-safe with `get()`. + +## Rules where behaviour is a judgement call + +- **Cardinals have no "and"**: "1,234" → "one thousand two hundred thirty-four". + British "one thousand two hundred **and** thirty-four" is not implemented. +- **Decimals read digit by digit** after "point" ("3.14" → "three point one four"). +- **Years**: a standalone 4-digit number in 1100–2099: 1100–1999 → + century + remainder ("nineteen ninety-nine", "nineteen oh five", + "nineteen hundred"); 2000–2009 → "two thousand five"; 2010–2099 → + "twenty twenty-six". 2100+ reads as an ordinary cardinal. +- **St.** is "saint" before a capitalised name, "street" otherwise; **Dr.** is + always "doctor" (drive/dr. disambiguation is out of scope, as the brief allows). +- **Acronyms**: unpronounceable all-caps words of 2–5 letters are spelled letter + by letter ("TDA" → T D A, "FBI" → F B I): a leading consonant cluster of 2+ or + an interior run of 3+ consonants makes an acronym unpronounceable, as does + having no vowels at all. Pronounceable ones (NASA, UNESCO and a small list) + stay words. Single letters are Word tokens that G2P should read as letter names. +- **a.m./p.m.** become the letter words "a m"/"p m". Lowercase "pm" without dots + is spelled; lowercase "am" is left alone (it's the English verb). +- **Fractions**: 1/2 → "one half"; other denominators ≤ 12 → "N ordinal(s)" + ("three quarters"); anything else → "N over M". Dates need a 4-digit year — + "10/7" alone reads as a fraction. +- **Digit groups**: hyphen-joined digits read digit by digit when any group has + a leading zero or there are 7+ digits in total ("555-0134"); otherwise two + small groups read "A to B" ("2-3" → "two to three"). +- **Pause markers**: `[pause s|ms]`, capped at 10 s; malformed markers + ([pause], [pause x], unclosed) stay plain text — never an error. +- **Money**: "$3.50" → "three dollars and fifty cents" (€ → euros/cents, £ → + pounds/pence); a lone `$`/`€`/`£` with no number stays a punctuation token. +- **Sentence ends**: runs of `.!?` are one `SentenceEnd`; abbreviation and + number dots are consumed by their rules and never end a sentence. + +## Limitations + +- English only (amended brief). Spanish was cut from this crate's scope. +- No "and" in British cardinals; no British-ism beyond date order and £. +- Pronunciation is not attempted here — this crate stops at spoken words. +- Time hours 0–12 read as cardinals ("0:15" → "zero fifteen"), not "twelve". +- Ordinal suffixes are validated but "1st"/"1 th"-style spacing is not accepted. +- "may" is never treated as a month abbreviation (it's a word); other dotted + month abbreviations (Jan.–Dec.) are. + +## References + +None (rules written from scratch). Clean-room: no TTS/normalizer source was read. + + +# P2b — English pronunciation (`phonemize`) + +```rust +pub fn phonemize(text: &str, lang: Lang, lex: &Lexicon, overrides: &Overrides) + -> Result, NormalizeError>; +pub enum Chunk { Phonemes(String), Pause { millis: u32 } } +pub const MAX_CHUNK_CHARS: usize = 400; +pub const SYMBOLS: &str; // every character phonemize may emit +``` + +`phonemize` normalizes the script, then phonemizes each sentence into one +`Chunk::Phonemes` (split at the last `,`/`;`/`:` before 400 chars, else at the +last space, else hard-split). Normalizer pauses become `Chunk::Pause`. One space +between words; punctuation from the normalizer attaches to the preceding word +with no space (`həlˈO, wˈɜɹld.`); a sentence terminator becomes a `.`. + +## The ARPAbet → Kokoro table (American English) + +AA→ɑ, AE→æ, AH0→ə, AH1/2→ʌ, AO→ɔ, AW→W, AY→I, EH→ɛ, ER0→ɚ, ER1/2→ɜɹ, EY→A, +IH→ɪ, IY→i, OW→O, OY→Y, UH→ʊ, UW→u; B→b, CH→ʧ, D→d, DH→ð, F→f, G→ɡ (U+0261, +not ASCII g), HH→h, JH→ʤ, K→k, L→l, M→m, N→n, NG→ŋ, P→p, R→ɹ, S→s, SH→ʃ, T→t, +TH→θ, V→v, W→w, Y→j, Z→z, ZH→ʒ. `A I O W Y` are Kokoro's single-symbol +diphthongs (eɪ aɪ oʊ aʊ ɔɪ). + +## Stress + +`ˈ` (primary) / `ˌ` (secondary) go immediately before the stressed vowel symbol, +not before the syllable: HELLO (HH AH0 L OW1) → `həlˈO`. Single-syllable +function words (a, an, the, of, to, and, in, is, it, for, on, with, at, by, as, +be, or, but, from, that, this) get no stress mark. Spelled-out single letters +use the letter's name from the dictionary (T → `tˈi`; CMUdict lists "A" as the +article first and the letter name as an alternate, and the letter-name alternate +is preferred). + +## Lookup order + +1. Overrides (case-insensitive; ≤ 1 000 entries of ≤ 200 chars honoured, excess + ignored). The value is either raw Kokoro phonemes between slashes + (`/tˌidˌiˈA/` — accepted only if every character is in `SYMBOLS`, otherwise + the override is ignored) or a plain-English respelling ("tee dee ay") that + goes through the same dictionary + rules. +2. Dictionary (first pronunciation). The dictionary is CMUdict in its text + format, parsed by `Lexicon::parse_cmudict` (alternates `WORD(2)`, `;;;` + comments, `#` trailing comments; malformed lines skipped and counted by + `skipped()`; input over 16 MiB or 500 000 entries is `LexError`). +3. Morphology on a dictionary stem: plural/possessive `-s`/`-'s` with voicing + (s / z / ɪz), `-ed` (t / d / ɪd), `-ing` (stem may regain its silent e: + making → make), `-er`, `-ly`, and the prefixes `un-`, `re-`. +4. Hyphenated words: each part goes through the same pipeline. +5. Letter-to-sound rules. + +## Letter-to-sound rules (written from scratch, clean-room) + +A left-to-right scanner with longest-match rules: `tion`/`sion` (stress on the +vowel before the suffix), `ture`/`sure`, `ough` (→ ɔ; +f word-final; "ought" → +ɔt), `igh`, `tch`, `dge`, `ch`, `sh`, `th`, `ph`, `ck`, `qu`, `wh`, `gh` (g at +the start, f word-final, silent before t), `ng` final, `nk`, initial `kn`/`gn` +(silent k/g), `wr`, final `mb`, `gu` (silent u), word-initial `ps`, `sch`, +`sten` (listen), final consonant+`le` → əl, vowel digraphs (ee/ea/ie→i, oo/ue/ui→u, +ou→W, ow→O, oi/oy→Y, ai/ay/ei→A, au/aw→ɔ, oa/oe→O), silent final e with +lengthening of the last vowel (cake → `kˈAk`), doubled consonants keeping the +vowel short (little → `lˈɪtəl`), soft c/g before e/i/y, `x` → ks, final y → i. +Primary stress lands on the first syllable unless a suffix rule moves it. +Every word with letters produces pronounceable output — never empty. + +## Measured accuracy + +Held-out measurement against the real CMUdict (`cmusphinx/cmudict`, commit +`74790861f652b15e4ac49015a90074ad62a27690`, SHA-256 +`81917843c7f44ce2b094ac63873c2c7a4cf802040792c455ba3ca406891c3d22`): every 50th +entry held out, lexicon rebuilt without them, letter-to-sound output compared +with the dictionary pronunciation. On 2026-10-08, machine: mjcamacho's Linux +workstation (rustc 1.99.0, release build): **229 held-out words, 9.1 % exact +word accuracy, 37.9 % phoneme error rate** (Levenshtein over phoneme symbols). +The test runs whenever `FILMCRAFT_CMUDICT` points at a CMUdict file and prints +the numbers; it asserts only that it runs. CMUdict is downloaded with the voices +at run time, never committed (BSD-style licence, attributed in the download +dialog). + +## Limitations + +- Homographs get their first CMUdict pronunciation only: "read" is always + /ɹiːd/, never /ɹɛd/; live/live, lead/lead, bow/bow are wrong half the time. + Overrides are the mitigation. +- Names, jargon and irregular spellings are the letter-to-sound rules' weak + spot (colonel, yacht, place names). Measured accuracy above is the honest + number. +- No flap-T or connected-speech effects ("water" is /ˈwɔtɚ/-style from the + dictionary, rules give ˈwɔtɚ only if the dictionary has it). +- `ow` is read /oʊ/ everywhere (so "wow" → wˈO), `ough` has one reading plus + word-final f; "ia"/"ea" ambiguities resolved to one value each. +- The sentence terminator always becomes `.` (the normalizer collapses `!`/`?` + into `SentenceEnd`). +- English only. + +## References + +None (rules written from scratch). Clean-room: no G2P/TTS project source was +read. Allowed sources used: CMUdict's own data and licence, published IPA +charts, and Wikipedia's ARPAbet and English phonology pages. diff --git a/crates/tts-text/src/lib.rs b/crates/tts-text/src/lib.rs new file mode 100644 index 000000000..0ef80e213 --- /dev/null +++ b/crates/tts-text/src/lib.rs @@ -0,0 +1,26 @@ +//! Text front end for FilmCraft's Text to Speech (phase P2a). +//! +//! [`normalize`] turns a narration script into plain spoken words, structured as +//! [`Token`]s: words, pauses ([`Token::Pause`], from `[pause 1s]`-style markers), +//! sentence ends ([`Token::SentenceEnd`]) and kept punctuation ([`Token::Punct`]). +//! It handles numbers (cardinals, ordinals, negatives, decimals), money (`$`, `€`, +//! `£`), percent, years, clock times, dates, phone-like digit groups, common +//! abbreviations, acronyms and the `&`/`+`/`@`/`#`/`/` symbols — English +//! ([`Lang::EnUs`] / [`Lang::EnGb`]). +//! +//! This crate is the first stage of the text pipeline; the grapheme-to-phoneme step +//! (P2b) consumes its words. It is pure Rust, depends only on `serde` and +//! `thiserror`, and never crashes on hostile input (see `AGENTS.md` §0): input over +//! 1 MiB is a [`NormalizeError::TooLong`], oversized numbers are read digit by digit, +//! and malformed pause markers fall back to plain text rather than erroring. +//! +//! The rules were written from scratch (clean-room); see the crate README for the +//! judgement calls and limitations. + +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + +mod normalize; +pub mod phonemize; + +pub use normalize::{Lang, NormalizeError, Token, normalize}; +pub use phonemize::{Chunk, LexError, Lexicon, MAX_CHUNK_CHARS, Overrides, SYMBOLS, phonemize}; diff --git a/crates/tts-text/src/normalize.rs b/crates/tts-text/src/normalize.rs new file mode 100644 index 000000000..7b5ee2c1a --- /dev/null +++ b/crates/tts-text/src/normalize.rs @@ -0,0 +1,1430 @@ +//! Text normalization: script → spoken words (see the crate docs). + +/// Language a script is normalized for. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Lang { + /// American English: month/day dates, "dollars and cents". + EnUs, + /// British English: day/month dates, "pounds and pence". + EnGb, +} + +/// A piece of a normalized script. +#[derive(Clone, Debug, PartialEq)] +pub enum Token { + /// A word to hand to G2P. Single uppercase letters are meant to be read as letter + /// names (spelled acronyms, a.m./p.m.); digit groups come through as digit words. + Word(String), + /// A narration pause, from `[pause 1s]` / `[pause 500ms]` / `[pause 1.5s]`. + Pause { + /// Pause length in milliseconds, capped at 10 s. + millis: u32, + }, + /// A sentence boundary ending in a period (never after an abbreviation or + /// inside a number). + SentenceEnd, + /// A sentence boundary with an explicit terminator (`?` or `!`), which Kokoro + /// reads for intonation. Added in M7.11; `SentenceEnd` keeps meaning `.`. + SentenceEndWith(char), + /// Punctuation kept for prosody downstream. + Punct(char), +} + +/// Normalization error. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum NormalizeError { + /// The script exceeds the 1 MiB input limit. + #[error("the script exceeds the 1 MiB normalization limit")] + TooLong, +} + +/// Maximum accepted input size (1 MiB). +const MAX_INPUT: usize = 1_048_576; +/// Maximum pause length (10 s). +const MAX_PAUSE_MS: u32 = 10_000; +/// Numbers with more than this many digits are read digit by digit (never overflow). +const MAX_CARDINAL_DIGITS: usize = 12; + +/// Normalize `text` into spoken-word tokens for `lang`. +/// +/// Never panics on any input: hostile input yields an `Err` or degraded (but valid) +/// tokens, never a crash. Malformed pause markers become plain text rather than an +/// error. Deterministic: the same input and language always give the same tokens. +pub fn normalize(text: &str, lang: Lang) -> Result, NormalizeError> { + if text.len() > MAX_INPUT { + return Err(NormalizeError::TooLong); + } + let mut out: Vec = Vec::new(); + let mut pos = 0usize; + while pos < text.len() { + if !text.is_char_boundary(pos) { + pos += 1; + continue; + } + let rest = text.get(pos..).unwrap_or(""); + let Some(c) = rest.chars().next() else { break }; + match c { + _ if c.is_whitespace() => pos += c.len_utf8(), + '[' => match parse_pause(rest) { + Some((millis, used)) => { + pos += used; + out.push(Token::Pause { millis }); + } + None => { + pos += c.len_utf8(); + out.push(Token::Punct(c)); + } + }, + '$' | '€' | '£' => match parse_money(rest, lang) { + Some((words, used)) => { + pos += used; + push_words(&mut out, &words); + } + None => { + pos += c.len_utf8(); + out.push(Token::Punct(c)); + } + }, + '-' if peek(rest, 1).is_some_and(|c| c.is_ascii_digit()) => { + pos += 1; + if let Some((tokens, used)) = parse_number_cluster(rest.get(1..).unwrap_or(""), lang) { + pos += used; + out.push(Token::Word("minus".into())); + out.extend(tokens); + } else { + out.push(Token::Punct(c)); + } + } + _ if c.is_ascii_digit() => match parse_number_cluster(rest, lang) { + Some((tokens, used)) => { + pos += used; + out.extend(tokens); + } + None => { + pos += c.len_utf8(); + out.push(Token::Punct(c)); + } + }, + '.' if peek(rest, 1).is_some_and(|c| c.is_ascii_digit()) => { + // A decimal with no integer part: ".5" → "point five". + pos += 1; + out.push(Token::Word("point".into())); + let (frac, used) = digit_run_as_words(rest.get(1..).unwrap_or("")); + pos += used; + push_words(&mut out, &frac); + } + '.' | '!' | '?' => { + // A run of terminators is one sentence end; the strongest one wins + // (`?` over `!` over `.`) so "Really?!" keeps its intonation mark. + // Abbreviations and numbers consume their own '.' earlier, so a + // leftover one ends a sentence. + let mut question = false; + let mut bang = false; + while matches!(peek(text.get(pos..).unwrap_or(""), 0), Some('.' | '!' | '?')) { + match peek(text.get(pos..).unwrap_or(""), 0) { + Some('?') => question = true, + Some('!') => bang = true, + _ => {} + } + pos += 1; + } + let terminator = if question { + Token::SentenceEndWith('?') + } else if bang { + Token::SentenceEndWith('!') + } else { + Token::SentenceEnd + }; + if !matches!(out.last(), Some(Token::SentenceEnd) | Some(Token::SentenceEndWith(_))) { + out.push(terminator); + } + } + _ if is_word_char(c) => match parse_wordish(rest, lang) { + Some((tokens, used)) => { + pos += used; + out.extend(tokens); + } + None => { + pos += c.len_utf8(); + out.push(Token::Punct(c)); + } + }, + '&' | '+' | '@' | '#' | '/' => { + // Symbols read as words. Slashes in dates and fractions are consumed + // by their parsers, so a leftover one is a real "slash". + pos += c.len_utf8(); + out.push(Token::Word(symbol_word(c).into())); + } + _ => { + pos += c.len_utf8(); + out.push(Token::Punct(c)); + } + } + } + Ok(out) +} + +fn push_words(out: &mut Vec, words: &[String]) { + for w in words { + out.push(Token::Word(w.clone())); + } +} + +fn is_word_char(c: char) -> bool { + c.is_alphabetic() || c == '\'' || c == '’' +} + +/// The char at byte offset `i` in `s`, if `i` is a char boundary inside `s`. +fn peek(s: &str, i: usize) -> Option { + s.get(i..).and_then(|r| r.chars().next()) +} + +// ---------------------------------------------------------------- pause markers + +/// `[pause 1s]`, `[pause 500ms]`, `[pause 1.5s]` → a capped pause. Malformed markers +/// return `None` and fall through to plain text (never an error). +fn parse_pause(rest: &str) -> Option<(u32, usize)> { + let close = rest.find(']')?; + let inner = rest.get(1..close)?.trim(); + let value = inner.strip_prefix("pause")?; + let value = value.trim_start(); + if value.is_empty() { + return None; + } + let mut saw_digit = false; + let mut saw_dot = false; + let mut num_end = 0usize; + for (i, ch) in value.char_indices() { + if ch.is_ascii_digit() { + saw_digit = true; + num_end = i + ch.len_utf8(); + } else if ch == '.' && saw_digit && !saw_dot { + saw_dot = true; + num_end = i + ch.len_utf8(); + } else { + num_end = i; + break; + } + } + if !saw_digit { + return None; + } + let number = value.get(..num_end)?; + let unit = value.get(num_end..)?.trim(); + let millis = pause_millis(number, unit)?; + Some((millis, close + 1)) +} + +/// `` in seconds or milliseconds, capped at [`MAX_PAUSE_MS`]. Overflow and +/// absurd values cap instead of erroring. +fn pause_millis(number: &str, unit: &str) -> Option { + let unit_ms: u64 = match unit { + "s" => 1000, + "ms" => 1, + _ => return None, + }; + let mut whole = 0u64; + let mut frac_digits: Vec = Vec::new(); + let mut after_dot = false; + let mut saw_digit = false; + for ch in number.chars() { + match ch { + '.' if !after_dot => after_dot = true, + '0'..='9' => { + saw_digit = true; + if after_dot { + if frac_digits.len() < 3 { + frac_digits.push(ch.to_digit(10)?); + } + } else { + whole = whole.checked_mul(10)?.checked_add(u64::from(ch.to_digit(10)?))?; + if whole > u64::from(MAX_PAUSE_MS) { + return Some(MAX_PAUSE_MS); + } + } + } + _ => return None, + } + } + if !saw_digit { + return None; + } + let mut millis = whole.checked_mul(unit_ms)?; + for (i, d) in frac_digits.iter().enumerate() { + let scale = unit_ms.checked_div(10u64.checked_pow(i as u32 + 1)?)?; + if scale == 0 { + break; + } + millis = millis.checked_add(u64::from(*d) * scale)?; + } + u32::try_from(millis).ok().map(|ms| ms.min(MAX_PAUSE_MS)) +} + +// ----------------------------------------------------------------------- money + +/// Parses a currency amount (the symbol is `rest`'s first char) into spoken words. +fn parse_money(rest: &str, lang: Lang) -> Option<(Vec, usize)> { + let symbol = peek(rest, 0)?; + let skip = symbol.len_utf8(); + let (value, digits, used, minor, had_minor) = parse_amount(rest.get(skip..)?, lang)?; + if digits > MAX_CARDINAL_DIGITS || has_leading_zero(&value, digits) { + // Oversized amounts: every digit, unit-free, no overflow. + let all = rest.get(skip..skip + used)?.chars().filter(|c| c.is_ascii_digit()).collect::(); + let (words, _) = digit_run_as_words(&all); + return Some((words, used + skip)); + } + let (unit, sub) = money_units(symbol, lang); + let mut words = english_cardinal(value); + push_unit(&mut words, unit, value == 1); + if had_minor && minor > 0 { + words.push("and".into()); + let mut cents = english_cardinal(u64::from(minor)); + push_unit(&mut cents, sub, minor == 1); + words.extend(cents); + } + Some((words, used + skip)) +} + +fn money_units(symbol: char, lang: Lang) -> (&'static str, &'static str) { + match (lang, symbol) { + (_, '$') => ("dollars", "cents"), + (_, '£') => ("pounds", "pence"), + (_, _) => ("euros", "cents"), + } +} + +fn push_unit(words: &mut Vec, unit: &str, singular: bool) { + if singular { + words.push(unit_singular(unit).into()); + } else { + words.push(unit.into()); + } +} + +fn unit_singular(unit: &str) -> &str { + match unit { + "dollars" => "dollar", + "euros" => "euro", + "pounds" => "pound", + "cents" => "cent", + "pence" => "penny", + _ => unit, + } +} + +/// Parses `[.]` honouring thousands grouping with commas. Returns the major +/// value, its digit count, bytes consumed, the minor value and whether one followed. +fn parse_amount(rest: &str, _lang: Lang) -> Option<(u64, usize, usize, u32, bool)> { + let mut int_digits = String::new(); + let mut frac_digits = String::new(); + let mut used = 0usize; + let mut after_decimal = false; + for (i, ch) in rest.char_indices() { + if ch.is_ascii_digit() { + if after_decimal { + if frac_digits.len() < 9 { + frac_digits.push(ch); + } + } else if int_digits.len() > MAX_CARDINAL_DIGITS { + break; + } else { + int_digits.push(ch); + } + used = i + 1; + } else if ch == ',' && !int_digits.is_empty() && !after_decimal { + if !grouping_followed_by_3(rest, i) { + break; + } + } else if ch == '.' && !after_decimal && !int_digits.is_empty() { + after_decimal = true; + used = i; + } else { + break; + } + } + if int_digits.is_empty() { + return None; + } + let major = int_digits.parse::().ok()?; + let had_minor = after_decimal && !frac_digits.is_empty(); + let minor = if had_minor { frac_digits.parse::().ok()? } else { 0 }; + Some((major, int_digits.len(), used, minor, had_minor)) +} + +/// A grouping mark is valid only when exactly three digits follow it. +fn grouping_followed_by_3(rest: &str, sep_index: usize) -> bool { + let after = rest.get(sep_index + 1..).unwrap_or(""); + let digits: String = after.chars().take_while(|c| c.is_ascii_digit()).collect(); + let boundary = after.get(digits.len()..).and_then(|r| r.chars().next()).is_none_or(|c| !c.is_ascii_digit()); + digits.len() == 3 && boundary +} + +fn has_leading_zero(value: &u64, digits: usize) -> bool { + digits > 1 && *value < 10u64.pow(digits as u32 - 1) +} + +// ---------------------------------------------------------------------- numbers + +/// Everything that starts with a digit: time, date, fraction, cardinal/year, +/// percent, ordinal, decimal, digit group. +fn parse_number_cluster(rest: &str, lang: Lang) -> Option<(Vec, usize)> { + if let Some(r) = parse_time(rest) { + return Some(r); + } + if let Some(r) = parse_date(rest, lang) { + return Some(r); + } + if let Some(r) = parse_fraction(rest) { + return Some(r); + } + parse_number_core(rest, lang) +} + +/// `3:30`, `15:05`, `03:30`, with an optional attached `am`/`pm`. +fn parse_time(rest: &str) -> Option<(Vec, usize)> { + let hour_end = digit_run(rest, 0, 2)?; + if peek(rest, hour_end) != Some(':') { + return None; + } + let minute_start = hour_end + 1; + let minute_end = digit_run(rest, minute_start, 2)?; + if minute_end - minute_start != 2 { + return None; + } + if peek(rest, minute_end).is_some_and(|c| c.is_ascii_digit()) { + return None; + } + let hour = rest.get(..hour_end)?.parse::().ok()?; + let minute = rest.get(minute_start..minute_end)?.parse::().ok()?; + if hour > 23 || minute > 59 { + return None; + } + let mut used = minute_end; + let mut tokens = time_words(hour, minute); + let after = rest.get(used..)?; + for (lit, letter_pair) in [("am", ['a', 'm']), ("pm", ['p', 'm'])] { + if after.get(..2).is_some_and(|s| s.eq_ignore_ascii_case(lit)) { + for letter in letter_pair { + tokens.push(Token::Word(letter.into())); + } + used += 2; + break; + } + } + Some((tokens, used)) +} + +/// End byte offset of a digit run starting at `start` (up to `max` digits). `None` +/// when there is no digit at `start`. +fn digit_run(s: &str, start: usize, max: usize) -> Option { + let sub = s.get(start..)?; + let end = sub.char_indices().take(max).take_while(|&(_, c)| c.is_ascii_digit()).map(|(i, c)| i + c.len_utf8()).last()?; + Some(start + end) +} + +fn time_words(h: u32, m: u32) -> Vec { + let mut out = vec![Token::Word(english_cardinal(u64::from(h)).join(" "))]; + if m == 0 { + out.push(Token::Word("zero".into())); + } else if m < 10 { + out.push(Token::Word("oh".into())); + out.push(Token::Word(english_digit_word(digit_char(m)).into())); + } else { + out.push(Token::Word(english_cardinal(u64::from(m)).join(" "))); + } + out +} + +fn digit_char(d: u32) -> char { + char::from_digit(d.min(9), 10).unwrap_or('0') +} + +/// `10/7/2026` (EnUs month/day) and `7/10/2026` (EnGb day/month) with a 4-digit year. +fn parse_date(rest: &str, lang: Lang) -> Option<(Vec, usize)> { + let (a, a_end) = read_u64(rest, 0, 2)?; + if peek(rest, a_end) != Some('/') { + return None; + } + let (b, b_end) = read_u64(rest, a_end + 1, 2)?; + if peek(rest, b_end) != Some('/') || !peek(rest, b_end + 1).is_some_and(|c| c.is_ascii_digit()) { + return None; + } + let (year, year_end) = read_u64(rest, b_end + 1, 4)?; + if year_end - (b_end + 1) != 4 || peek(rest, year_end).is_some_and(|c| c.is_ascii_digit()) { + return None; + } + let (month, day) = match lang { + Lang::EnUs => (a, b), + Lang::EnGb => (b, a), + }; + if !(1..=12).contains(&month) || !(1..=31).contains(&day) { + return None; + } + let mut tokens = date_words(month, day, lang); + tokens.extend(year_words(year, lang)); + Some((tokens, year_end)) +} + +fn read_u64(s: &str, start: usize, max: usize) -> Option<(u64, usize)> { + let end = digit_run(s, start, max)?; + if end == start { + return None; + } + let digits = s.get(start..end)?; + if digits.len() > 19 { + return None; + } + digits.parse::().ok().map(|v| (v, end)) +} + +fn date_words(month: u64, day: u64, lang: Lang) -> Vec { + let mut out = Vec::new(); + let month_name = english_month(month).unwrap_or("month"); + match lang { + Lang::EnUs => { + out.push(Token::Word(month_name.into())); + out.push(Token::Word(english_ordinal(day))); + } + Lang::EnGb => { + out.push(Token::Word("the".into())); + out.push(Token::Word(english_ordinal(day))); + out.push(Token::Word("of".into())); + out.push(Token::Word(month_name.into())); + } + } + out +} + +fn english_month(month: u64) -> Option<&'static str> { + Some(match month { + 1 => "January", + 2 => "February", + 3 => "March", + 4 => "April", + 5 => "May", + 6 => "June", + 7 => "July", + 8 => "August", + 9 => "September", + 10 => "October", + 11 => "November", + 12 => "December", + _ => return None, + }) +} + +/// `1/2` → "one half"; other small denominators read "N ordinal(s)", anything +/// bigger reads "N over M". +fn parse_fraction(rest: &str) -> Option<(Vec, usize)> { + let (n, n_end) = read_u64(rest, 0, 2)?; + if peek(rest, n_end) != Some('/') || n == 0 { + return None; + } + let (d, d_end) = read_u64(rest, n_end + 1, 2)?; + if d == 0 || peek(rest, d_end).is_some_and(|c| c.is_ascii_alphanumeric() || c == '/') { + return None; + } + let mut words = english_cardinal(n); + if d > 12 { + // "7/32" → "seven over thirty-two". + words.push("over".into()); + words.extend(english_cardinal(d)); + return Some((words.into_iter().map(Token::Word).collect(), d_end)); + } + match d { + 2 => words.push(if n > 1 { "halves".into() } else { "half".into() }), + 4 => words.push(if n > 1 { "quarters".into() } else { "quarter".into() }), + _ => { + let mut denom = english_ordinal(d); + if n > 1 { + denom.push('s'); + } + words.push(denom); + } + } + Some((words.into_iter().map(Token::Word).collect(), d_end)) +} + +/// Integer (grouped), then the continuations: `%`, ordinal, decimal, digit group — +/// or a plain cardinal / year. +fn parse_number_core(rest: &str, lang: Lang) -> Option<(Vec, usize)> { + let mut digits = String::new(); + let mut used = 0usize; + let mut overflowed = false; + let mut grouped = false; + for (i, ch) in rest.char_indices() { + if ch.is_ascii_digit() { + digits.push(ch); + used = i + 1; + if digits.len() > 19 { + overflowed = true; + break; + } + } else if ch == ',' && !digits.is_empty() { + if grouping_followed_by_3(rest, i) { + // Grouping is visual only; drop the mark. + used = i + 1; + grouped = true; + } else { + break; + } + } else { + break; + } + } + if digits.is_empty() { + return None; + } + if overflowed || digits.len() > MAX_CARDINAL_DIGITS { + let words: Vec = digits.chars().map(|c| Token::Word(english_digit_word(c).into())).collect(); + return Some((words, used)); + } + let value: u64 = digits.parse().ok()?; + if let Some((tokens, u)) = parse_continuation(rest, used, value) { + return Some((tokens, u)); + } + if has_leading_zero_str(&digits) { + let words: Vec = digits.chars().map(|c| Token::Word(english_digit_word(c).into())).collect(); + return Some((words, used)); + } + // A year is a bare, ungrouped 4-digit number standing alone. + if !grouped && digits.len() == 4 && (1100..=2099).contains(&value) && year_is_standalone(rest, used) { + return Some((year_words(value, lang), used)); + } + Some((english_cardinal(value).into_iter().map(Token::Word).collect(), used)) +} + +fn has_leading_zero_str(digits: &str) -> bool { + digits.len() > 1 && digits.starts_with('0') +} + +/// `%`, an ordinal suffix (`st`/`nd`/`rd`/`th`), a decimal fraction or `-`-joined +/// digit groups. +fn parse_continuation(rest: &str, used: usize, value: u64) -> Option<(Vec, usize)> { + let after = rest.get(used..)?; + match peek(after, 0) { + Some('%') => { + let mut words = english_cardinal(value); + words.push("percent".into()); + Some((words.into_iter().map(Token::Word).collect(), used + 1)) + } + Some('s' | 'S' | 't' | 'T' | 'n' | 'N' | 'd' | 'D' | 'r' | 'R') => { + let suffix = after.get(..2)?; + if is_ordinal_suffix(suffix) && !peek(after, 2).is_some_and(|c| c.is_alphabetic()) { + return Some((vec![Token::Word(english_ordinal(value))], used + 2)); + } + None + } + Some('.') if peek(after, 1).is_some_and(|c| c.is_ascii_digit()) => { + let mut words = english_cardinal(value); + words.push("point".into()); + let (frac, frac_used) = digit_run_as_words(after.get(1..)?); + if frac.is_empty() { + return None; + } + words.extend(frac); + Some((words.into_iter().map(Token::Word).collect(), used + 1 + frac_used)) + } + Some('-') => parse_digit_groups(rest, used), + _ => None, + } +} + +/// `555-0134` → digit words (any group with a leading zero, or 7+ digits in total); +/// otherwise `A-B` with two small groups reads "A to B". +fn parse_digit_groups(rest: &str, used: usize) -> Option<(Vec, usize)> { + let after = rest.get(used..)?.strip_prefix('-')?; + let mut groups: Vec = Vec::new(); + let mut current = String::new(); + let mut end = used + 1; + for (i, ch) in after.char_indices() { + if ch.is_ascii_digit() { + current.push(ch); + end = used + 1 + i + 1; + } else if ch == '-' && !current.is_empty() { + groups.push(std::mem::take(&mut current)); + end = used + 1 + i + 1; + } else { + break; + } + } + if !current.is_empty() { + groups.push(current); + } + if groups.iter().any(String::is_empty) { + return None; + } + if groups.len() == 1 { + let group = groups.first()?; + let leading_zero = group.starts_with('0'); + // Total digits across the whole cluster: the integer before the dash plus this group. + let total = rest.get(..used).map_or(0, |head| head.chars().filter(char::is_ascii_digit).count()) + group.len(); + if !leading_zero && total < 7 { + // "2-3" → "two to three". + let b = group.parse::().ok()?; + let a = rest.get(..used)?.parse::().ok()?; + let mut words = english_cardinal(a); + words.push("to".into()); + words.extend(english_cardinal(b)); + return Some((words.into_iter().map(Token::Word).collect(), end)); + } + let mut words: Vec = Vec::new(); + // The integer before the dash read digit by digit too. + if let Some(head) = rest.get(..used) { + for ch in head.chars() { + if ch.is_ascii_digit() { + words.push(Token::Word(english_digit_word(ch).into())); + } + } + } + for ch in group.chars() { + words.push(Token::Word(english_digit_word(ch).into())); + } + return Some((words, end)); + } + let total: usize = groups.iter().map(String::len).sum::() + rest.get(..used).map_or(0, |head| head.chars().filter(char::is_ascii_digit).count()); + let leading_zero = groups.iter().any(|g| g.starts_with('0')); + if total >= 7 || leading_zero { + let mut words: Vec = Vec::new(); + if let Some(head) = rest.get(..used) { + for ch in head.chars() { + if ch.is_ascii_digit() { + words.push(Token::Word(english_digit_word(ch).into())); + } + } + } + for g in &groups { + for ch in g.chars() { + words.push(Token::Word(english_digit_word(ch).into())); + } + } + return Some((words, end)); + } + if groups.len() == 2 && groups.iter().all(|g| g.len() <= 3) { + let a = rest.get(..used)?.parse::().ok()?; + let b_str = groups.last()?; + let b = b_str.parse::().ok()?; + let mut words = english_cardinal(a); + words.push("to".into()); + words.extend(english_cardinal(b)); + return Some((words.into_iter().map(Token::Word).collect(), end)); + } + None +} + +fn is_ordinal_suffix(suffix: &str) -> bool { + matches!(suffix.to_ascii_lowercase().as_str(), "st" | "nd" | "rd" | "th") +} + +/// A 4-digit number reads as a year when nothing alphanumeric follows. Any +/// separator that continues a number (decimal point with digits, grouping comma +/// with three digits, time colon, date slash) has already been consumed by its +/// own parser before this check, so a leftover `.`/`,`/`:`/`/` — or a sentence +/// terminator or closing bracket — is fine. +fn year_is_standalone(rest: &str, used: usize) -> bool { + !peek(rest, used).is_some_and(|c| c.is_alphanumeric()) +} + +/// A digit run read digit by digit. +fn digit_run_as_words(s: &str) -> (Vec, usize) { + let mut words = Vec::new(); + let mut used = 0usize; + for (i, ch) in s.char_indices() { + if !ch.is_ascii_digit() { + break; + } + words.push(english_digit_word(ch).into()); + used = i + 1; + } + (words, used) +} + +fn english_digit_word(c: char) -> &'static str { + match c { + '0' => "zero", + '1' => "one", + '2' => "two", + '3' => "three", + '4' => "four", + '5' => "five", + '6' => "six", + '7' => "seven", + '8' => "eight", + '9' => "nine", + _ => "", + } +} + +// ------------------------------------------------------------- cardinal numbers + +/// English cardinal words for values up to 999 999 999 999 (callers cap the digit +/// count before calling). No "and" (British style is a documented limitation). +fn english_cardinal(value: u64) -> Vec { + if value == 0 { + return vec!["zero".into()]; + } + let scales = ["", "thousand", "million", "billion", "trillion"]; + let mut groups: Vec = Vec::new(); + let mut n = value; + while n > 0 { + groups.push(n % 1000); + n /= 1000; + } + let mut words: Vec = Vec::new(); + for (i, g) in groups.iter().enumerate().rev() { + if *g == 0 { + continue; + } + words.extend(english_below_thousand(*g)); + if i > 0 + && let Some(scale) = scales.get(i) + { + words.push((*scale).into()); + } + } + words +} + +fn english_below_thousand(n: u64) -> Vec { + let mut words: Vec = Vec::new(); + let hundreds = n / 100; + let rest = n % 100; + if hundreds > 0 { + words.push(english_digit_word(digit_char(hundreds as u32 % 10)).into()); + words.push("hundred".into()); + } + if rest > 0 { + words.extend(english_below_hundred(rest)); + } + words +} + +fn english_below_hundred(n: u64) -> Vec { + const UNITS: [&str; 20] = [ + "zero", + "one", + "two", + "three", + "four", + "five", + "six", + "seven", + "eight", + "nine", + "ten", + "eleven", + "twelve", + "thirteen", + "fourteen", + "fifteen", + "sixteen", + "seventeen", + "eighteen", + "nineteen", + ]; + const TENS: [&str; 10] = ["", "", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"]; + if n < 20 { + return vec![UNITS.get(n as usize).copied().unwrap_or("").into()]; + } + let tens = TENS.get((n / 10) as usize).copied().unwrap_or(""); + let units = n % 10; + if units == 0 { vec![tens.into()] } else { vec![format!("{tens}-{}", UNITS.get(units as usize).copied().unwrap_or(""))] } +} + +/// English ordinal: "7th" → "seventh", "21st" → "twenty-first", "100th" → "one hundredth". +fn english_ordinal(value: u64) -> String { + let mut words = english_cardinal(value); + let Some(last) = words.pop() else { return "zeroth".into() }; + let stem = words.join(" "); + if matches!(last.as_str(), "hundred" | "thousand" | "million" | "billion" | "trillion") { + // The scale word takes the ordinal form: "one hundred" → "one hundredth". + return if stem.is_empty() { format!("{last}th") } else { format!("{stem} {last}th") }; + } + let (prefix, tail) = match last.rsplit_once('-') { + Some((p, t)) => (format!("{p}-"), t), + None => (String::new(), last.as_str()), + }; + let tail = match tail { + "one" => "first", + "two" => "second", + "three" => "third", + "five" => "fifth", + "eight" => "eighth", + "nine" => "ninth", + "twelve" => "twelfth", + t if t.ends_with('y') => { + let stem = t.get(..t.len() - 1).unwrap_or(t); + return format!("{prefix}{stem}ieth"); + } + t => return attach(stem, &format!("{prefix}{t}th")), + }; + attach(stem, &format!("{prefix}{tail}")) +} + +fn attach(stem: String, s: &str) -> String { + if stem.is_empty() { s.to_string() } else { format!("{stem} {s}") } +} + +/// English year readings: 1999 → "nineteen ninety-nine", 1905 → "nineteen oh five", +/// 1900 → "nineteen hundred", 2005 → "two thousand five", 2026 → "twenty twenty-six". +fn year_words(year: u64, _lang: Lang) -> Vec { + let mut out: Vec = Vec::new(); + if (2000..=2009).contains(&year) { + out.push(Token::Word("two".into())); + out.push(Token::Word("thousand".into())); + if !year.is_multiple_of(10) { + out.push(Token::Word(english_digit_word(digit_char((year % 10) as u32)).into())); + } + return out; + } + let century = year / 100; + let rest = year % 100; + for w in english_cardinal(century) { + out.push(Token::Word(w)); + } + if rest == 0 { + out.push(Token::Word("hundred".into())); + } else if rest < 10 { + out.push(Token::Word("oh".into())); + out.push(Token::Word(english_digit_word(digit_char(rest as u32)).into())); + } else { + for w in english_cardinal(rest) { + out.push(Token::Word(w)); + } + } + out +} + +// ----------------------------------------------------------------- words, etc. + +/// Words, abbreviations, months, acronyms, a.m./p.m. and e.g./i.e. +fn parse_wordish(rest: &str, _lang: Lang) -> Option<(Vec, usize)> { + let collected: String = rest.chars().take_while(|c| is_word_char(*c)).collect(); + let word = collected.trim_matches(|c| c == '\'' || c == '’'); + if word.is_empty() { + return None; + } + let used = word.len() + leading_apostrophes(rest); + let lower = word.to_ascii_lowercase(); + + // a.m. / p.m. — a single letter, a dot, "m", and a dot or boundary. + if word.len() == 1 && matches!(lower.as_str(), "a" | "p") && peek(rest, used) == Some('.') && matches!(peek(rest, used + 1), Some('m') | Some('M')) { + let after_m = used + 2; + if peek(rest, after_m).is_none_or(|c| !c.is_alphanumeric()) { + let mut tokens = vec![Token::Word(word.to_ascii_lowercase())]; + tokens.push(Token::Word("m".into())); + let consumed = if peek(rest, after_m) == Some('.') { after_m + 1 } else { after_m }; + return Some((tokens, consumed)); + } + } + // e.g. / i.e. — a single letter, a dot, a letter, a dot. + if word.len() == 1 + && matches!(lower.as_str(), "e" | "i") + && peek(rest, used) == Some('.') + && let Some(second) = peek(rest, used + 1) + { + let pair = format!("{lower}{second}"); + if matches!(pair.as_str(), "eg" | "ie") && peek(rest, used + 2) == Some('.') { + let expansion = if pair == "eg" { "for example" } else { "that is" }; + return Some((vec![Token::Word(expansion.into())], used + 3)); + } + } + // Dotted abbreviations: Mr. Dr. St. Mt. vs. etc., and months (Oct. 7th). + if let Some((tokens, consumed)) = dotted_abbreviation(&lower, rest, used) { + return Some((tokens, consumed)); + } + // Standalone "pm" (no dots) is spelled; lowercase "am" is a real word, so it stays. + if lower == "pm" { + return Some((vec![Token::Word("p".into()), Token::Word("m".into())], used)); + } + // All-caps 2–5 letters with no vowels are spelled letter by letter (TDA, NBC); + // pronounceable ones (with vowels, e.g. NASA) stay words. + if is_spellable_acronym(word) { + let tokens = word.chars().map(|c| Token::Word(c.to_string())).collect(); + return Some((tokens, used)); + } + Some((vec![Token::Word(word.into())], used)) +} + +fn leading_apostrophes(rest: &str) -> usize { + rest.chars().take_while(|c| *c == '\'' || *c == '’').map(char::len_utf8).sum() +} + +/// Expansions for `Mr.` etc.; the dot is consumed so it never ends a sentence. +/// `St.` picks "saint" before a capitalised name and "street" otherwise. +fn dotted_abbreviation(lower: &str, rest: &str, used: usize) -> Option<(Vec, usize)> { + if peek(rest, used) != Some('.') { + return None; + } + let expansion: Vec = match lower { + "mr" => vec![Token::Word("mister".into())], + "mrs" => vec![Token::Word("missus".into())], + "dr" => vec![Token::Word("doctor".into())], + "mt" => vec![Token::Word("mount".into())], + "st" => { + let mut i = used + 1; + while peek(rest, i).is_some_and(|c| c.is_whitespace()) { + i += 1; + } + let capitalised = peek(rest, i).is_some_and(|c| c.is_uppercase()); + vec![Token::Word(if capitalised { "saint".into() } else { "street".into() })] + } + "vs" => vec![Token::Word("versus".into())], + "etc" => vec![Token::Word("et cetera".into())], + _ => { + let month = match lower { + "jan" => "January", + "feb" => "February", + "mar" => "March", + "apr" => "April", + "jun" => "June", + "jul" => "July", + "aug" => "August", + "sep" | "sept" => "September", + "oct" => "October", + "nov" => "November", + "dec" => "December", + _ => return None, + }; + vec![Token::Word(month.into())] + } + }; + Some((expansion, used + 1)) +} + +/// All-caps, 2–5 ASCII letters that are not pronounceable → spell it out. +/// "Pronounceable" here: no consonant cluster of 3+, and no leading cluster of 2+ +/// before the first vowel (TDA, NBC, FBI, HTML are spelled; NASA, UNESCO stay words). +/// A small allowlist of known pronounceable acronyms is kept as words regardless. +fn is_spellable_acronym(word: &str) -> bool { + const KEEP_AS_WORDS: [&str; 8] = ["NASA", "UNESCO", "NATO", "UNICEF", "OPEC", "FIFA", "RADAR", "SONAR"]; + let letters: Vec = word.chars().collect(); + if letters.len() < 2 || letters.len() > 5 || !letters.iter().all(|c| c.is_ascii_uppercase()) { + return false; + } + if KEEP_AS_WORDS.contains(&word) { + return false; + } + if !letters.iter().any(|c| is_vowel(*c)) { + return true; + } + // Leading consonant cluster of 2+ → hard to pronounce → spell. + let mut leading_consonants = 0usize; + for c in &letters { + if is_vowel(*c) { + break; + } + leading_consonants += 1; + } + if leading_consonants >= 2 { + return true; + } + // Any interior run of 3+ consonants → spell. + let mut run = 0usize; + for c in &letters { + if is_vowel(*c) { + run = 0; + } else { + run += 1; + if run >= 3 { + return true; + } + } + } + false +} + +fn is_vowel(c: char) -> bool { + matches!(c, 'A' | 'E' | 'I' | 'O' | 'U') +} + +fn symbol_word(c: char) -> &'static str { + match c { + '&' => "and", + '+' => "plus", + '@' => "at", + '#' => "number", + '/' => "slash", + _ => "", + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Flatten tokens the way the tests compare them: words joined by spaces, + /// pauses as `[pause N]`, sentence ends as `|`, punctuation as the raw char. + fn flat(tokens: &[Token]) -> String { + let mut s = String::new(); + for t in tokens { + match t { + Token::Word(w) => { + if !s.is_empty() { + s.push(' '); + } + s.push_str(w); + } + Token::Pause { millis } => { + if !s.is_empty() { + s.push(' '); + } + s.push_str(&format!("[pause {millis}]")); + } + Token::SentenceEnd => { + if !s.is_empty() { + s.push(' '); + } + s.push('|'); + } + Token::SentenceEndWith(c) => { + if !s.is_empty() { + s.push(' '); + } + s.push('|'); + s.push(*c); + } + Token::Punct(c) => { + if !s.is_empty() { + s.push(' '); + } + s.push(*c); + } + } + } + s + } + + fn words(text: &str, lang: Lang) -> String { + flat(&normalize(text, lang).unwrap()) + } + + #[test] + fn plain_words_pass_through() { + assert_eq!(words("Hello world", Lang::EnUs), "Hello world"); + assert_eq!(words("well-known", Lang::EnUs), "well - known"); + assert_eq!(words("it's fine", Lang::EnUs), "it's fine"); + } + + #[test] + fn cardinals() { + assert_eq!(words("0", Lang::EnUs), "zero"); + assert_eq!(words("7", Lang::EnUs), "seven"); + assert_eq!(words("13", Lang::EnUs), "thirteen"); + assert_eq!(words("20", Lang::EnUs), "twenty"); + assert_eq!(words("34", Lang::EnUs), "thirty-four"); + assert_eq!(words("100", Lang::EnUs), "one hundred"); + assert_eq!(words("105", Lang::EnUs), "one hundred five"); + assert_eq!(words("999", Lang::EnUs), "nine hundred ninety-nine"); + assert_eq!(words("1,234", Lang::EnUs), "one thousand two hundred thirty-four"); + assert_eq!(words("1,234,567", Lang::EnUs), "one million two hundred thirty-four thousand five hundred sixty-seven"); + assert_eq!(words("1000000", Lang::EnUs), "one million"); + assert_eq!(words("1000000000", Lang::EnUs), "one billion"); + assert_eq!( + words("999,999,999,999", Lang::EnUs), + "nine hundred ninety-nine billion nine hundred ninety-nine million nine hundred ninety-nine thousand nine hundred ninety-nine" + ); + assert_eq!( + words("123456789012", Lang::EnUs), + "one hundred twenty-three billion four hundred fifty-six million seven hundred eighty-nine thousand twelve" + ); + } + + #[test] + fn oversized_and_leading_zero_numbers_go_digit_by_digit() { + assert_eq!(words("007", Lang::EnUs), "zero zero seven"); + assert_eq!(words("12345678901234", Lang::EnUs), "one two three four five six seven eight nine zero one two three four"); + assert_eq!(words("0.5", Lang::EnUs), "zero point five"); + } + + #[test] + fn negatives_and_decimals() { + assert_eq!(words("-5", Lang::EnUs), "minus five"); + assert_eq!(words("3.14", Lang::EnUs), "three point one four"); + assert_eq!(words(".5", Lang::EnUs), "point five"); + assert_eq!(words("-2.5", Lang::EnUs), "minus two point five"); + } + + #[test] + fn ordinals() { + assert_eq!(words("1st", Lang::EnUs), "first"); + assert_eq!(words("2nd", Lang::EnUs), "second"); + assert_eq!(words("3rd", Lang::EnUs), "third"); + assert_eq!(words("7th", Lang::EnUs), "seventh"); + assert_eq!(words("9th", Lang::EnUs), "ninth"); + assert_eq!(words("12th", Lang::EnUs), "twelfth"); + assert_eq!(words("20th", Lang::EnUs), "twentieth"); + assert_eq!(words("21st", Lang::EnUs), "twenty-first"); + assert_eq!(words("33rd", Lang::EnUs), "thirty-third"); + assert_eq!(words("100th", Lang::EnUs), "one hundredth"); + assert_eq!(words("101st", Lang::EnUs), "one hundred first"); + } + + #[test] + fn money() { + assert_eq!(words("$3.50", Lang::EnUs), "three dollars and fifty cents"); + assert_eq!(words("$1", Lang::EnUs), "one dollar"); + assert_eq!(words("$1.00", Lang::EnUs), "one dollar"); + assert_eq!(words("$5.05", Lang::EnUs), "five dollars and five cents"); + assert_eq!(words("$0.75", Lang::EnUs), "zero dollars and seventy-five cents"); + assert_eq!(words("$1,234.56", Lang::EnUs), "one thousand two hundred thirty-four dollars and fifty-six cents"); + assert_eq!(words("€2", Lang::EnUs), "two euros"); + assert_eq!(words("€1", Lang::EnUs), "one euro"); + assert_eq!(words("£2.50", Lang::EnUs), "two pounds and fifty pence"); + assert_eq!(words("£1", Lang::EnUs), "one pound"); + assert_eq!(words("$0.50", Lang::EnUs), "zero dollars and fifty cents"); + } + + #[test] + fn percent() { + assert_eq!(words("20%", Lang::EnUs), "twenty percent"); + assert_eq!(words("100%", Lang::EnUs), "one hundred percent"); + assert_eq!(words("7%", Lang::EnUs), "seven percent"); + } + + #[test] + fn years() { + assert_eq!(words("1999", Lang::EnUs), "nineteen ninety-nine"); + assert_eq!(words("2005", Lang::EnUs), "two thousand five"); + assert_eq!(words("2026", Lang::EnUs), "twenty twenty-six"); + assert_eq!(words("1900", Lang::EnUs), "nineteen hundred"); + assert_eq!(words("1905", Lang::EnUs), "nineteen oh five"); + assert_eq!(words("1100", Lang::EnUs), "eleven hundred"); + assert_eq!(words("2000", Lang::EnUs), "two thousand"); + // Not years: outside the range, or part of a bigger number. + assert_eq!(words("2100", Lang::EnUs), "two thousand one hundred"); + assert_eq!(words("1099", Lang::EnUs), "one thousand ninety-nine"); + assert_eq!(words("19995", Lang::EnUs), "nineteen thousand nine hundred ninety-five"); + } + + #[test] + fn times() { + assert_eq!(words("3:30", Lang::EnUs), "three thirty"); + assert_eq!(words("15:05", Lang::EnUs), "fifteen oh five"); + assert_eq!(words("0:15", Lang::EnUs), "zero fifteen"); + assert_eq!(words("03:30", Lang::EnUs), "three thirty"); + assert_eq!(words("23:59", Lang::EnUs), "twenty-three fifty-nine"); + assert_eq!(words("3:30pm", Lang::EnUs), "three thirty p m"); + assert_eq!(words("3:30 p.m.", Lang::EnUs), "three thirty p m"); + assert_eq!(words("9:00 a.m.", Lang::EnUs), "nine zero a m"); + } + + #[test] + fn dates() { + assert_eq!(words("10/7/2026", Lang::EnUs), "October seventh twenty twenty-six"); + assert_eq!(words("7/10/2026", Lang::EnGb), "the seventh of October twenty twenty-six"); + assert_eq!(words("Oct. 7th", Lang::EnUs), "October seventh"); + assert_eq!(words("Jan. 1st", Lang::EnUs), "January first"); + assert_eq!(words("Sept. 3rd", Lang::EnUs), "September third"); + assert_eq!(words("May 5th", Lang::EnUs), "May fifth"); + assert_eq!(words("October 7th, 2026", Lang::EnUs), "October seventh , twenty twenty-six"); + assert_eq!(words("9/9/1999", Lang::EnUs), "September ninth nineteen ninety-nine"); + assert_eq!(words("13/7/2026", Lang::EnUs), "thirteen slash seven slash twenty twenty-six"); + assert_eq!(words("10/7/2026.", Lang::EnUs), "October seventh twenty twenty-six |"); + } + + #[test] + fn digit_groups() { + assert_eq!(words("555-0134", Lang::EnUs), "five five five zero one three four"); + assert_eq!(words("555-0134-22", Lang::EnUs), "five five five zero one three four two two"); + assert_eq!(words("0800-100", Lang::EnUs), "zero eight zero zero one zero zero"); + assert_eq!(words("2-3", Lang::EnUs), "two to three"); + assert_eq!(words("10-12", Lang::EnUs), "ten to twelve"); + } + + #[test] + fn abbreviations() { + assert_eq!(words("Mr. Smith", Lang::EnUs), "mister Smith"); + assert_eq!(words("Mrs. Lee", Lang::EnUs), "missus Lee"); + assert_eq!(words("Dr. Smith", Lang::EnUs), "doctor Smith"); + assert_eq!(words("St. James", Lang::EnUs), "saint James"); + assert_eq!(words("James St.", Lang::EnUs), "James street"); + assert_eq!(words("Mt. Fuji", Lang::EnUs), "mount Fuji"); + assert_eq!(words("vs. foo", Lang::EnUs), "versus foo"); + assert_eq!(words("etc. and", Lang::EnUs), "et cetera and"); + assert_eq!(words("e.g. x", Lang::EnUs), "for example x"); + assert_eq!(words("i.e. y", Lang::EnUs), "that is y"); + // An abbreviation's dot never ends the sentence. + let tokens = normalize("Call Dr. Smith. Now.", Lang::EnUs).unwrap(); + assert_eq!(flat(&tokens), "Call doctor Smith | Now |"); + } + + #[test] + fn acronyms() { + assert_eq!(words("TDA", Lang::EnUs), "T D A"); + assert_eq!(words("NBC", Lang::EnUs), "N B C"); + assert_eq!(words("FBI", Lang::EnUs), "F B I"); + assert_eq!(words("HTML", Lang::EnUs), "H T M L"); + assert_eq!(words("NASA", Lang::EnUs), "NASA"); + assert_eq!(words("UNESCO", Lang::EnUs), "UNESCO"); + assert_eq!(words("laser", Lang::EnUs), "laser"); + assert_eq!(words("A", Lang::EnUs), "A"); + } + + #[test] + fn symbols() { + assert_eq!(words("&", Lang::EnUs), "and"); + assert_eq!(words("+", Lang::EnUs), "plus"); + assert_eq!(words("@", Lang::EnUs), "at"); + assert_eq!(words("#", Lang::EnUs), "number"); + assert_eq!(words("/", Lang::EnUs), "slash"); + assert_eq!(words("#3", Lang::EnUs), "number three"); + assert_eq!(words("Tom & Jerry", Lang::EnUs), "Tom and Jerry"); + } + + #[test] + fn fractions() { + assert_eq!(words("1/2", Lang::EnUs), "one half"); + assert_eq!(words("3/4", Lang::EnUs), "three quarters"); + assert_eq!(words("2/3", Lang::EnUs), "two thirds"); + assert_eq!(words("1/3", Lang::EnUs), "one third"); + assert_eq!(words("22/7", Lang::EnUs), "twenty-two sevenths"); + assert_eq!(words("5/8", Lang::EnUs), "five eighths"); + assert_eq!(words("7/32", Lang::EnUs), "seven over thirty-two"); + } + + #[test] + fn pause_markers() { + let tokens = normalize("[pause 1s]", Lang::EnUs).unwrap(); + assert_eq!(tokens, vec![Token::Pause { millis: 1000 }]); + assert_eq!(words("[pause 500ms]", Lang::EnUs), "[pause 500]"); + assert_eq!(words("[pause 1.5s]", Lang::EnUs), "[pause 1500]"); + assert_eq!(words("[pause 10s]", Lang::EnUs), "[pause 10000]"); + assert_eq!(words("[pause 0.25s]", Lang::EnUs), "[pause 250]"); + assert_eq!(words("[pause 11s]", Lang::EnUs), "[pause 10000]"); + assert_eq!(words("[pause 999999999999s]", Lang::EnUs), "[pause 10000]"); + // Malformed markers stay plain text. + assert_eq!(words("[pause", Lang::EnUs), "[ pause"); + assert_eq!(words("[pause x]", Lang::EnUs), "[ pause x ]"); + assert_eq!(words("[pause 5]", Lang::EnUs), "[ pause five ]"); + assert_eq!(words("[pause]", Lang::EnUs), "[ pause ]"); + } + + #[test] + fn sentence_ends() { + let tokens = normalize("Hi. Bye! What?", Lang::EnUs).unwrap(); + assert_eq!(flat(&tokens), "Hi | Bye |! What |?"); + // Runs of terminators are one end; the strongest terminator wins. + assert_eq!(flat(&normalize("Wow!!!", Lang::EnUs).unwrap()), "Wow |!"); + assert_eq!(flat(&normalize("Really?! Yes.", Lang::EnUs).unwrap()), "Really |? Yes |"); + // The terminator is carried through for intonation (M7.11 fix round). + assert_eq!(flat(&normalize("Go!", Lang::EnUs).unwrap()), "Go |!"); + assert_eq!(flat(&normalize("Done.", Lang::EnUs).unwrap()), "Done |"); + assert_eq!(flat(&normalize("Is it 1999?", Lang::EnUs).unwrap()), "Is it nineteen ninety-nine |?"); + assert_eq!(flat(&normalize("Call 555-0134!", Lang::EnUs).unwrap()), "Call five five five zero one three four |!"); + assert_eq!(flat(&normalize("Wait... what?", Lang::EnUs).unwrap()), "Wait | what |?"); + assert_eq!(flat(&normalize("No!!", Lang::EnUs).unwrap()), "No |!"); + assert_eq!(flat(&normalize("Are you there?!", Lang::EnUs).unwrap()), "Are you there |?"); + assert_eq!(flat(&normalize("Yes?", Lang::EnGb).unwrap()), "Yes |?"); + // A decimal point is not a sentence end; an abbreviation's dot is consumed. + assert_eq!(flat(&normalize("It costs 3.14 dollars.", Lang::EnUs).unwrap()), "It costs three point one four dollars |"); + assert_eq!(flat(&normalize("Mr. Smith called.", Lang::EnUs).unwrap()), "mister Smith called |"); + // No sentence end inside "1,234." until the real end. + assert_eq!(flat(&normalize("Total: 1,234.", Lang::EnUs).unwrap()), "Total : one thousand two hundred thirty-four |"); + } + + #[test] + fn the_briefs_example_sentence() { + assert_eq!( + words("Call 555-0134 by 3:30 p.m. on Oct. 7th, it's $3.50 (20% off)", Lang::EnUs), + "Call five five five zero one three four by three thirty p m on October seventh , it's three dollars and fifty cents ( twenty percent off )" + ); + } + + #[test] + fn punctuation_is_kept() { + assert_eq!(words("a, b; c", Lang::EnUs), "a , b ; c"); + assert_eq!(words("(x)", Lang::EnUs), "( x )"); + assert_eq!(words("a—b", Lang::EnUs), "a — b"); + assert_eq!(words("end.", Lang::EnUs), "end |"); + } + + #[test] + fn bad_grouping_falls_back() { + assert_eq!(words("1,23", Lang::EnUs), "one , twenty-three"); + assert_eq!(words("1,2345", Lang::EnUs), "one , two thousand three hundred forty-five"); + } + + #[test] + fn hostile_inputs_do_not_panic() { + let hostile: Vec = vec![ + String::new(), + " \t\n ".into(), + "x".repeat(MAX_INPUT) + "x", + "9".repeat(10_000), + "$$$$".into(), + "[pause 999999999999s]".into(), + "[pause".into(), + "[pause x]".into(), + "日本語のテキスト".into(), + "emoji 👍👍👍".into(), + "العربية".into(), + "\u{0301}".into(), + "\0".into(), + "3:".into(), + "1,,2".into(), + ".....".into(), + "[[[[".into(), + "-".into(), + "$".into(), + "3:305".into(), + "5-".into(), + "999999999999999999999999".into(), + ]; + for h in &hostile { + let result = std::panic::catch_unwind(|| normalize(h, Lang::EnUs)); + assert!(result.is_ok(), "panicked on {h:?}"); + // The oversized case must error, not degrade. + if h.len() > MAX_INPUT { + assert_eq!(result.unwrap(), Err(NormalizeError::TooLong)); + } + } + } + + #[test] + fn deterministic() { + let samples = ["Call 555-0134 by 3:30 p.m. on Oct. 7th, it's $3.50 (20% off)", "1,234", "[pause 1s]", "TDA & NASA"]; + for s in samples { + let a = normalize(s, Lang::EnUs).unwrap(); + let b = normalize(s, Lang::EnUs).unwrap(); + assert_eq!(a, b); + let c = normalize(s, Lang::EnGb).unwrap(); + assert_eq!(flat(&a), flat(&c), "langs only change word choices, not structure"); + } + } + + #[test] + fn british_vs_american() { + assert_eq!(words("£2.50", Lang::EnGb), "two pounds and fifty pence"); + assert_eq!(words("7/10/2026", Lang::EnGb), "the seventh of October twenty twenty-six"); + assert_eq!(words("10/7/2026", Lang::EnUs), "October seventh twenty twenty-six"); + assert_eq!(words("1,234", Lang::EnGb), "one thousand two hundred thirty-four"); + } + + #[test] + fn sentence_final_numbers_end_the_sentence() { + assert_eq!(words("in 1999.", Lang::EnUs), "in nineteen ninety-nine |"); + assert_eq!(words("costs $1,234,567.", Lang::EnUs), "costs one million two hundred thirty-four thousand five hundred sixty-seven dollars |"); + assert_eq!(words("It was 3.5 percent.", Lang::EnUs), "It was three point five percent |"); + assert_eq!(words("Version 2. Next.", Lang::EnUs), "Version two | Next |"); + assert_eq!(words("Call 555-0134.", Lang::EnUs), "Call five five five zero one three four |"); + assert_eq!(words("At 3:30.", Lang::EnUs), "At three thirty |"); + assert_eq!(words("On 10/7/2026.", Lang::EnUs), "On October seventh twenty twenty-six |"); + assert_eq!(words("Up 20%.", Lang::EnUs), "Up twenty percent |"); + // The review's two failing sentences. + assert_eq!(words("Dr. Smith lives on Main St. in 1999.", Lang::EnUs), "doctor Smith lives on Main street in nineteen ninety-nine |"); + assert_eq!( + words("The TDA project for NASA costs $1,234,567.", Lang::EnUs), + "The T D A project for NASA costs one million two hundred thirty-four thousand five hundred sixty-seven dollars |" + ); + } + + #[test] + fn year_rule_survives_punctuation() { + assert_eq!(words("1999.", Lang::EnUs), "nineteen ninety-nine |"); + assert_eq!(words("1999,", Lang::EnUs), "nineteen ninety-nine ,"); + assert_eq!(words("1999!", Lang::EnUs), "nineteen ninety-nine |!"); + assert_eq!(words("(1999)", Lang::EnUs), "( nineteen ninety-nine )"); + assert_eq!(words("in 1999.", Lang::EnGb), "in nineteen ninety-nine |"); + } +} diff --git a/crates/tts-text/src/phonemize.rs b/crates/tts-text/src/phonemize.rs new file mode 100644 index 000000000..6d6c1f33d --- /dev/null +++ b/crates/tts-text/src/phonemize.rs @@ -0,0 +1,1489 @@ +//! English pronunciation: spoken words → Kokoro phoneme symbols (task M7.11). +//! +//! [`phonemize`] normalizes a script ([`crate::normalize`]) and turns each word into +//! Kokoro's phoneme symbols: pronunciation overrides first, then a CMUdict-format +//! [`Lexicon`] (downloaded with the voices at run time, never committed), then +//! morphology on dictionary stems (plural/possessive `-s`, `-ed`, `-ing`, `-er`, +//! `-ly`, `un-`, `re-`), then letter-to-sound rules written from scratch. +//! +//! Output alphabet: only the characters in [`SYMBOLS`] — the ARPAbet→Kokoro +//! mapping below, the stress marks `ˈ` (primary) and `ˌ` (secondary), spaces +//! between words, and a small punctuation set. Claude's Kokoro code checks +//! against [`SYMBOLS`], so nothing outside it may ever be emitted. +//! +//! ARPAbet → Kokoro (American English): AA→ɑ, AE→æ, AH0→ə, AH1/2→ʌ, AO→ɔ, AW→W, +//! AY→I, EH→ɛ, ER0→ɚ, ER1/2→ɜɹ, EY→A, IH→ɪ, IY→i, OW→O, OY→Y, UH→ʊ, UW→u, +//! B→b, CH→ʧ, D→d, DH→ð, F→f, G→ɡ (U+0261), HH→h, JH→ʤ, K→k, L→l, M→m, N→n, +//! NG→ŋ, P→p, R→ɹ, S→s, SH→ʃ, T→t, TH→θ, V→v, W→w, Y→j, Z→z, ZH→ʒ. +//! `A I O W Y` are Kokoro's single-symbol diphthongs (eɪ aɪ oʊ aʊ ɔɪ). + +use crate::normalize::{self, Lang, NormalizeError, Token}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Maximum characters in one [`Chunk::Phonemes`]; longer sentences are split. +pub const MAX_CHUNK_CHARS: usize = 400; +/// Maximum dictionary input size (16 MiB). +const MAX_DICT_BYTES: usize = 16 * 1024 * 1024; +/// Maximum number of dictionary entries. +const MAX_DICT_ENTRIES: usize = 500_000; +/// Maximum number of overrides honoured (excess ignored, not an error). +const MAX_OVERRIDES: usize = 1_000; +/// Maximum length of one override value (longer ones ignored). +const MAX_OVERRIDE_LEN: usize = 200; + +/// Every character [`phonemize`] may ever emit: the phoneme symbols, the two +/// stress marks, one space, and the kept punctuation. Claude's vocabulary check +/// is exactly this set. +pub const SYMBOLS: &str = "ˈˌ ɑæəʌɔɛɚɜɪʊiunAIOWYbʧdðfɡhʤklmŋpɹsʃtθvwjzʒ,.!?;:—…()\""; + +/// Punctuation the normalizer may emit that is kept in phoneme output; anything +/// else (apostrophes, stray symbols) is dropped. +const KEPT_PUNCT: [char; 11] = [',', '.', '!', '?', ';', ':', '—', '…', '(', ')', '"']; + +/// A pronunciation dictionary in CMUdict format. +#[derive(Clone, Debug, Default)] +pub struct Lexicon { + /// Uppercase word → its pronunciations in ARPAbet (first is used for lookup). + entries: BTreeMap>, + /// Lines skipped because they were malformed. + skipped: usize, +} + +/// Dictionary parse errors. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum LexError { + /// The input exceeds the 16 MiB limit. + #[error("the pronunciation dictionary exceeds the 16 MiB limit")] + TooLarge, + /// The dictionary has more than 500 000 entries. + #[error("the pronunciation dictionary has more than 500 000 entries")] + TooManyEntries, +} + +impl Lexicon { + /// An empty dictionary (everything falls through to letter-to-sound rules). + pub fn empty() -> Lexicon { + Lexicon::default() + } + + /// Parse CMUdict's text format: `WORD P1 P2 …`, alternates as `WORD(2)`, + /// `;;;` comment lines and ` #` trailing comments. Malformed lines are + /// skipped and counted by [`Lexicon::skipped`], never an error. Refuses + /// input over 16 MiB or with more than 500 000 entries ([`LexError`]). + pub fn parse_cmudict(bytes: &[u8]) -> Result { + if bytes.len() > MAX_DICT_BYTES { + return Err(LexError::TooLarge); + } + let text = String::from_utf8_lossy(bytes); + let mut entries: BTreeMap> = BTreeMap::new(); + let mut skipped = 0usize; + for line in text.lines() { + let line = line.trim(); + if line.is_empty() || line.starts_with(";;;") { + continue; + } + // A trailing ` #` comment; words themselves never contain ` #`. + let had_comment = line.contains(" #"); + let code = match line.find(" #") { + Some(i) => line.get(..i).unwrap_or(line), + None => line, + }; + let code = code.trim(); + if code.is_empty() { + continue; + } + let mut parts = code.split_whitespace(); + let Some(head) = parts.next() else { + skipped += 1; + continue; + }; + let phones: Vec<&str> = parts.collect(); + if phones.is_empty() { + if had_comment { + continue; + } + skipped += 1; + continue; + } + // `WORD(2)` alternates fold into the same key. + let key = match (head.find('('), head.rfind(')')) { + (Some(open), Some(close)) if close > open + 1 && head.get(open + 1..close).is_some_and(|d| d.chars().all(|c| c.is_ascii_digit())) => { + head.get(..open).unwrap_or(head) + } + (Some(_), _) => { + skipped += 1; + continue; + } + (None, _) => head, + }; + if phones.iter().any(|p| !is_arpabet_token(p)) { + skipped += 1; + continue; + } + if !entries.contains_key(key) && entries.len() >= MAX_DICT_ENTRIES { + return Err(LexError::TooManyEntries); + } + let key = key.to_ascii_uppercase(); + entries.entry(key).or_default().push(phones.join(" ")); + } + Ok(Lexicon { entries, skipped }) + } + + /// Number of distinct words in the dictionary. + pub fn len(&self) -> usize { + self.entries.len() + } + + /// Number of malformed lines skipped during parsing. + pub fn skipped(&self) -> usize { + self.skipped + } + + /// Whether the dictionary is empty. + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + /// All pronunciations of a word (already uppercase), first one first. + fn pronunciations(&self, word_upper: &str) -> Option<&Vec> { + self.entries.get(word_upper) + } +} + +/// One ARPAbet token: a known symbol with an optional stress digit. +fn is_arpabet_token(token: &str) -> bool { + let (letters, digits) = token.split_at(token.find(|c: char| c.is_ascii_digit()).unwrap_or(token.len())); + if !matches!(digits, "" | "0" | "1" | "2") { + return false; + } + matches!( + letters, + "AA" | "AE" + | "AH" + | "AO" + | "AW" + | "AY" + | "B" + | "CH" + | "D" + | "DH" + | "EH" + | "ER" + | "EY" + | "F" + | "G" + | "HH" + | "IH" + | "IY" + | "JH" + | "K" + | "L" + | "M" + | "N" + | "NG" + | "OW" + | "OY" + | "P" + | "R" + | "S" + | "SH" + | "T" + | "TH" + | "UH" + | "UW" + | "V" + | "W" + | "Y" + | "Z" + | "ZH" + ) +} + +/// The user's pronunciation list: word (case-insensitive) → how it should sound. +/// The value is either a respelling in plain English ("tee dee ay"), which goes +/// through the same dictionary + rules, or raw Kokoro phonemes between slashes +/// ("/tˌidˌiˈA/"). +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct Overrides(pub BTreeMap); + +/// A piece of the phonemized script. +#[derive(Clone, Debug, PartialEq)] +pub enum Chunk { + /// Kokoro phonemes for a stretch of speech, at most [`MAX_CHUNK_CHARS`] chars. + Phonemes(String), + /// A narration pause, straight from the normalizer's `[pause …]` markers. + Pause { + /// Pause length in milliseconds. + millis: u32, + }, +} + +/// Script → chunks: normalize, then phonemize each sentence (one +/// [`Chunk::Phonemes`] per sentence, split when over [`MAX_CHUNK_CHARS`]). +/// Never crashes; hostile input degrades (AGENTS.md §0). +pub fn phonemize(text: &str, lang: Lang, lex: &Lexicon, overrides: &Overrides) -> Result, NormalizeError> { + let tokens = normalize::normalize(text, lang)?; + let overrides = bounded_overrides(overrides); + let mut chunks: Vec = Vec::new(); + let mut sentence: Vec = Vec::new(); + for token in tokens { + match token { + Token::SentenceEnd => { + // The sentence terminator attaches to the last word as a period. + sentence.push(Token::Punct('.')); + flush_sentence(&mut sentence, &mut chunks, lex, &overrides, lang); + } + Token::SentenceEndWith(c) => { + // `?` and `!` shape Kokoro's intonation; keep them. + sentence.push(Token::Punct(c)); + flush_sentence(&mut sentence, &mut chunks, lex, &overrides, lang); + } + Token::Pause { millis } => { + flush_sentence(&mut sentence, &mut chunks, lex, &overrides, lang); + chunks.push(Chunk::Pause { millis }); + } + other => sentence.push(other), + } + } + flush_sentence(&mut sentence, &mut chunks, lex, &overrides, lang); + Ok(chunks) +} + +/// The bounded view of the user's overrides: keys lowercased, at most +/// [`MAX_OVERRIDES`] entries of at most [`MAX_OVERRIDE_LEN`] chars — excess is +/// ignored, not an error. +fn bounded_overrides(overrides: &Overrides) -> BTreeMap { + let mut out = BTreeMap::new(); + for (word, value) in overrides.0.iter() { + if out.len() >= MAX_OVERRIDES { + break; + } + if value.chars().count() > MAX_OVERRIDE_LEN { + continue; + } + out.insert(word.to_lowercase(), value.clone()); + } + out +} + +/// Phonemize one buffered sentence into zero or more `Phonemes` chunks. +fn flush_sentence(sentence: &mut Vec, chunks: &mut Vec, lex: &Lexicon, overrides: &BTreeMap, lang: Lang) { + let tokens = std::mem::take(sentence); + let mut out = String::new(); + for token in &tokens { + match token { + Token::Word(w) => { + let Some(p) = word_phonemes(w, lex, Some(overrides), lang) else { continue }; + if !out.is_empty() { + out.push(' '); + } + out.push_str(&p); + } + Token::Punct(c) => { + if KEPT_PUNCT.contains(c) { + out.push(*c); + } + } + Token::Pause { .. } | Token::SentenceEnd | Token::SentenceEndWith(_) => {} + } + } + for piece in split_long_chunk(out) { + if !piece.is_empty() { + chunks.push(Chunk::Phonemes(piece)); + } + } +} + +/// Split a phoneme string longer than [`MAX_CHUNK_CHARS`] at the last `,` `;` `:` +/// before the limit, else at the last space, else hard-split at the limit. +fn split_long_chunk(s: String) -> Vec { + if s.chars().count() <= MAX_CHUNK_CHARS { + return vec![s]; + } + let limit = s.char_indices().map(|(i, _)| i).nth(MAX_CHUNK_CHARS).unwrap_or(s.len()); + let head_boundary = s.get(..limit).and_then(|head| { + head.char_indices().rev().find(|&(_, c)| matches!(c, ',' | ';' | ':')).map(|(i, _)| i + 1).or_else(|| head.rfind(' ').map(|i| i + 1)).or(Some(limit)) + }); + let Some(cut) = head_boundary else { return vec![s] }; + let head = s.get(..cut).unwrap_or("").trim_end().to_string(); + let tail = s.get(cut..).unwrap_or("").trim_start().to_string(); + let mut out = vec![head]; + out.extend(split_long_chunk(tail)); + out.retain(|p| !p.is_empty()); + out +} + +/// Whether `c` may appear in raw-phoneme override values. +fn is_output_char(c: char) -> bool { + SYMBOLS.contains(c) +} + +/// Phonemes for one word: overrides → dictionary → morphology → letter-to-sound. +/// `overrides` is `None` while expanding a respelling, so overrides can't loop. +fn word_phonemes(word: &str, lex: &Lexicon, overrides: Option<&BTreeMap>, lang: Lang) -> Option { + let lower = word.to_lowercase(); + if lower.is_empty() { + return None; + } + if let Some(map) = overrides + && let Some(value) = map.get(&lower) + { + if value.starts_with('/') { + // A raw-phoneme override; an invalid one is ignored entirely. + if let Some(raw) = raw_override(value) { + return Some(raw); + } + } else { + // A respelling: plain English words through dictionary + rules. + let tokens = normalize::normalize(value, lang).ok()?; + let mut out = String::new(); + for token in tokens { + if let Token::Word(w) = token + && let Some(p) = word_phonemes(&w, lex, None, Lang::EnUs) + { + if !out.is_empty() { + out.push(' '); + } + out.push_str(&p); + } + } + if !out.is_empty() { + return Some(out); + } + } + } + // Spelled-out single letters use the letter's name; a lowercase single + // letter that is a function word (a, i) stays on the normal path. + if is_single_letter(word) { + return Some(letter_phonemes(word, lex)); + } + // Hyphenated words: each part goes through the same pipeline. + if word.contains('-') && !word.contains(char::is_numeric) { + let mut out = String::new(); + for part in word.split('-') { + if let Some(p) = word_phonemes(part, lex, None, lang) { + if !out.is_empty() { + out.push(' '); + } + out.push_str(&p); + } + } + if !out.is_empty() { + return Some(out); + } + } + let upper = lower.to_uppercase(); + if let Some(pron) = lex.pronunciations(&upper) + && let Some(first) = pron.first() + { + return Some(arpabet_to_kokoro(first, is_function_word(&lower))); + } + if let Some(p) = morphology(&lower, lex) { + return Some(p); + } + letter_to_sound(&lower) +} + +/// A raw-phoneme override `/…/`: strip the slashes and accept it only when every +/// character is in [`SYMBOLS`]; otherwise fall through to the respelling path. +fn raw_override(value: &str) -> Option { + let inner = value.strip_prefix('/')?.strip_suffix('/')?; + if inner.is_empty() || !inner.chars().all(is_output_char) { + return None; + } + Some(inner.to_string()) +} + +fn is_single_letter(word: &str) -> bool { + let mut chars = word.chars(); + matches!((chars.next(), chars.next()), (Some(c), None) if c.is_ascii_alphabetic()) + && !(word.chars().all(char::is_lowercase) && is_function_word(&word.to_lowercase())) +} + +/// English letter names in ARPAbet (public knowledge, not copied from any +/// project); used when the dictionary has no entry for the letter. +const LETTER_NAMES: [(&str, &str); 26] = [ + ("A", "EY1"), + ("B", "B IY1"), + ("C", "S IY1"), + ("D", "D IY1"), + ("E", "IY1"), + ("F", "EH F1"), + ("G", "JH IY1"), + ("H", "EY CH1"), + ("I", "AY1"), + ("J", "JH EY1"), + ("K", "K EY1"), + ("L", "EH L1"), + ("M", "EH M1"), + ("N", "EH N1"), + ("O", "OW1"), + ("P", "P IY1"), + ("Q", "K Y UW1"), + ("R", "AA R1"), + ("S", "EH S1"), + ("T", "T IY1"), + ("U", "Y UW1"), + ("V", "V IY1"), + ("W", "D AH1 B AH0 L Y UW1"), + ("X", "EH K S1"), + ("Y", "W AY1"), + ("Z", "Z IY1"), +]; + +/// Phonemes for a spelled-out single uppercase letter: prefer the dictionary's +/// entry that matches the letter name (CMUdict lists "A" as the article first, +/// the letter name second), else the first entry, else the built-in name. +fn letter_phonemes(word: &str, lex: &Lexicon) -> String { + let upper = word.to_uppercase(); + let letter_name = LETTER_NAMES.iter().find(|(l, _)| *l == upper).map(|(_, p)| *p); + if let Some(pron) = lex.pronunciations(&upper) { + if let Some(name) = letter_name + && let Some(matched) = pron.iter().find(|p| p.as_str() == name) + { + return arpabet_to_kokoro(matched, false); + } + if let Some(first) = pron.first() { + return arpabet_to_kokoro(first, false); + } + } + if let Some(name) = letter_name { + return arpabet_to_kokoro(name, false); + } + String::new() +} + +/// Single-syllable function words carry no stress mark. +fn is_function_word(lower: &str) -> bool { + matches!( + lower, + "a" | "an" + | "the" + | "of" + | "to" + | "and" + | "in" + | "is" + | "it" + | "for" + | "on" + | "with" + | "at" + | "by" + | "as" + | "be" + | "or" + | "but" + | "from" + | "that" + | "this" + ) +} + +/// One ARPAbet phoneme with its stress digit (if any). +#[derive(Clone, Debug, PartialEq, Eq)] +struct Phone { + symbol: String, + stress: Option, +} + +/// Split an ARPAbet pronunciation into phonemes. +fn parse_arpabet(pron: &str) -> Vec { + pron.split_whitespace() + .map(|token| { + let cut = token.find(|c: char| c.is_ascii_digit()).unwrap_or(token.len()); + let symbol = match token.get(..cut) { + Some(s) => s, + None => token, + }; + let stress = token.get(cut..).and_then(|d| d.parse::().ok()); + Phone { symbol: symbol.to_string(), stress } + }) + .collect() +} + +/// Kokoro symbol for an ARPAbet phoneme (stress handled by the caller). +fn kokoro_symbol(symbol: &str, stress: Option) -> &'static str { + match (symbol, stress) { + ("AH", Some(0)) => "ə", + ("ER", Some(0)) => "ɚ", + ("AA", _) => "ɑ", + ("AE", _) => "æ", + ("AH", _) => "ʌ", + ("AO", _) => "ɔ", + ("AW", _) => "W", + ("AY", _) => "I", + ("EH", _) => "ɛ", + ("ER", _) => "ɜɹ", + ("EY", _) => "A", + ("IH", _) => "ɪ", + ("IY", _) => "i", + ("OW", _) => "O", + ("OY", _) => "Y", + ("UH", _) => "ʊ", + ("UW", _) => "u", + ("B", _) => "b", + ("CH", _) => "ʧ", + ("D", _) => "d", + ("DH", _) => "ð", + ("F", _) => "f", + ("G", _) => "ɡ", + ("HH", _) => "h", + ("JH", _) => "ʤ", + ("K", _) => "k", + ("L", _) => "l", + ("M", _) => "m", + ("N", _) => "n", + ("NG", _) => "ŋ", + ("P", _) => "p", + ("R", _) => "ɹ", + ("S", _) => "s", + ("SH", _) => "ʃ", + ("T", _) => "t", + ("TH", _) => "θ", + ("V", _) => "v", + ("W", _) => "w", + ("Y", _) => "j", + ("Z", _) => "z", + ("ZH", _) => "ʒ", + _ => "", + } +} + +/// Whether an ARPAbet symbol is a vowel (carries stress). +fn is_vowel_symbol(symbol: &str) -> bool { + matches!(symbol, "AA" | "AE" | "AH" | "AO" | "AW" | "AY" | "EH" | "ER" | "EY" | "IH" | "IY" | "OW" | "OY" | "UH" | "UW") +} + +/// ARPAbet → Kokoro symbols, with `ˈ`/`ˌ` immediately before the stressed vowel +/// symbol (or no marks for function words). +fn arpabet_to_kokoro(pron: &str, strip_stress: bool) -> String { + let mut out = String::new(); + for phone in parse_arpabet(pron) { + let symbol = kokoro_symbol(&phone.symbol, phone.stress); + if symbol.is_empty() { + continue; + } + if !strip_stress && is_vowel_symbol(&phone.symbol) { + match phone.stress { + Some(1) => out.push('ˈ'), + Some(2) => out.push('ˌ'), + _ => {} + } + } + out.push_str(symbol); + } + out +} + +// ------------------------------------------------------------------ morphology + +/// Suffix/prefix morphology on dictionary stems: plural/possessive `-s`/`-'s` +/// with voicing (s / z / ɪz), `-ed` (t / d / ɪd), `-ing`, `-er`, `-ly`, and the +/// prefixes `un-`, `re-`. Only fires when the stem is in the dictionary. +fn morphology(word: &str, lex: &Lexicon) -> Option { + // Possessive / plural. + for stem in [strip_suffix(word, "'s"), strip_suffix(word, "s"), strip_suffix(word, "es")].into_iter().flatten() { + if let Some(pron) = lex.pronunciations(&stem.to_uppercase()) + && let Some(first) = pron.first() + && let Some(last) = parse_arpabet(first).last() + { + let s = plural_s(last); + return Some(join_phonemes(&parse_arpabet(first), &[s])); + } + } + // -ed. + for stem in [strip_suffix(word, "ed"), strip_suffix(word, "d")].into_iter().flatten() { + if let Some(pron) = lex.pronunciations(&stem.to_uppercase()) + && let Some(first) = pron.first() + && let Some(last) = parse_arpabet(first).last() + { + let ed = past_ed(last); + return Some(join_phonemes(&parse_arpabet(first), &[ed])); + } + } + // -ing (stem may need its silent e back: making → make). + let ing_stems: [Option; 2] = [strip_suffix(word, "ing").map(|s| s.to_string()), strip_suffix(word, "ing").map(|s| format!("{s}e"))]; + for stem in ing_stems.into_iter().flatten() { + let stem = stem.as_str(); + if let Some(pron) = lex.pronunciations(&stem.to_uppercase()) + && let Some(first) = pron.first() + { + return Some(join_phonemes(&parse_arpabet(first), &["IH", "NG"])); + } + } + // -er, -ly. + for (suffix, added) in [("er", ["ER0"].as_slice()), ("ly", ["L", "IY0"].as_slice())] { + if let Some(stem) = strip_suffix(word, suffix) + && let Some(pron) = lex.pronunciations(&stem.to_uppercase()) + && let Some(first) = pron.first() + { + return Some(join_phonemes(&parse_arpabet(first), added)); + } + } + // Prefixes un-, re-. + for (prefix, added) in [("un", ["AH0", "N"]), ("re", ["R", "IY0"])] { + if let Some(stem) = word.strip_prefix(prefix) + && !stem.is_empty() + && let Some(pron) = lex.pronunciations(&stem.to_uppercase()) + && let Some(first) = pron.first() + { + let mut phones: Vec = added.iter().flat_map(|s| parse_arpabet(s)).collect(); + phones.extend(parse_arpabet(first)); + return Some(join_phonemes(&phones, &[])); + } + } + None +} + +/// Strip `suffix` (lowercase) from the end of a lowercase word. +fn strip_suffix<'a>(word: &'a str, suffix: &str) -> Option<&'a str> { + let stem = word.strip_suffix(suffix)?; + if stem.is_empty() { + return None; + } + Some(stem) +} + +/// Join phonemes into Kokoro symbols; the suffix phonemes get no stress marks +/// (they are unstressed in these endings). +fn join_phonemes(stem: &[Phone], suffix: &[&str]) -> String { + let mut out = arpabet_to_kokoro_phones(stem, false); + for group in suffix { + for phone in parse_arpabet(group) { + out.push_str(kokoro_symbol(&phone.symbol, phone.stress)); + } + } + out +} + +/// ARPAbet phones → Kokoro with stress marks. +fn arpabet_to_kokoro_phones(phones: &[Phone], strip_stress: bool) -> String { + let pron = phones + .iter() + .map(|p| match p.stress { + Some(s) => format!("{}{s}", p.symbol), + None => p.symbol.to_string(), + }) + .collect::>() + .join(" "); + arpabet_to_kokoro(&pron, strip_stress) +} + +/// The `-s`/`-'s` allomorph in ARPAbet: ɪz after sibilants, s after voiceless +/// sounds, z elsewhere. +fn plural_s(last: &Phone) -> &'static str { + match last.symbol.as_str() { + "S" | "Z" | "SH" | "ZH" | "CH" | "JH" => "IH0 Z", + "P" | "T" | "K" | "F" | "TH" => "S", + _ => "Z", + } +} + +/// The `-ed` allomorph: ɪd after t/d, t after voiceless sounds, d elsewhere. +fn past_ed(last: &Phone) -> &'static str { + match last.symbol.as_str() { + "T" | "D" => "IH0 D", + "P" | "K" | "F" | "TH" | "S" | "SH" | "CH" => "T", + _ => "D", + } +} + +// -------------------------------------------------------- letter-to-sound rules + +/// Letter-to-sound rules for words the dictionary doesn't have, written from +/// scratch. Silent final e, vowel digraphs, `tion`/`sion`, `ph`/`gh`/`ck`/`qu`, +/// doubled consonants, soft c/g, `y` as vowel; primary stress on the first +/// syllable (on the vowel before -tion/-sion). Always pronounceable. +fn letter_to_sound(word: &str) -> Option { + let letters: Vec = word.chars().filter(|c| c.is_ascii_alphabetic()).collect(); + if letters.is_empty() { + return None; + } + // -ture / -sure endings (nature, measure): the e belongs to the suffix, so it + // is handled before the silent-e rule. + if letters.len() >= 5 { + let tail_chars = letters.get(letters.len() - 4..)?; + let tail: String = tail_chars.iter().collect(); + if tail == "ture" || tail == "sure" { + let head_chars = letters.get(..letters.len() - 4)?; + let head: String = head_chars.iter().collect(); + let mut body = letter_to_sound(&head).unwrap_or_default(); + if !body.is_empty() && !body.contains('ˈ') { + // mark the head's first vowel + if let Some(pos) = body.chars().collect::>().iter().position(|c| is_vowel_kokoro(&c.to_string())) { + let mut v: Vec = body.chars().collect(); + v.insert(pos, 'ˈ'); + body = v.into_iter().collect(); + } + } + body.push_str(if tail == "ture" { "ʧɚ" } else { "ʒɚ" }); + return Some(body); + } + } + // Silent final e: drop it and lengthen the last vowel ("cake" → kˈA k). + let mut chars = letters.clone(); + let mut long_last = false; + let mut last_vowel_index: Option = None; + let ends_consonant_le = chars.len() >= 3 + && chars.last() == Some(&'e') + && chars.get(chars.len() - 2) == Some(&'l') + && chars.get(chars.len() - 3).is_some_and(|c| !is_vowel_letter(*c)); + if !ends_consonant_le && chars.len() >= 3 && chars.last() == Some(&'e') && chars.get(chars.len() - 2).is_some_and(|c| !is_vowel_letter(*c)) { + chars.pop(); + long_last = true; + last_vowel_index = chars.iter().rposition(|c| is_vowel_letter(*c)); + } + let n = chars.len(); + let mut out: Vec = Vec::new(); + let mut vowel_positions: Vec = Vec::new(); + let mut i = 0usize; + while i < n { + let Some(rest) = chars.get(i..) else { break }; + let rest_str: String = rest.iter().collect(); + let at_start = i == 0; + // Multi-letter rules first (longest match wins). + if rest_str.ends_with("tion") && rest_str.len() == 4 { + // The vowel before the suffix takes the primary stress. + if let Some(&last) = vowel_positions.last() + && let Some(v) = out.get_mut(last) + { + *v = format!("ˈ{v}"); + } + out.push("ʃ".into()); + out.push("ə".into()); + out.push("n".into()); + break; + } + if rest_str.ends_with("sion") && rest_str.len() == 4 { + if let Some(&last) = vowel_positions.last() + && let Some(v) = out.get_mut(last) + { + *v = format!("ˈ{v}"); + } + out.push("ʒ".into()); + out.push("ə".into()); + out.push("n".into()); + break; + } + if rest_str.starts_with("sch") { + out.push("s".into()); + out.push("k".into()); + i += 3; + continue; + } + if rest_str == "sten" { + // listen + out.push("s".into()); + out.push("ə".into()); + out.push("n".into()); + break; + } + if rest_str == "gue" { + out.push("ɡ".into()); + break; + } + if rest_str == "que" { + out.push("k".into()); + break; + } + if rest_str.starts_with("ough") { + // A famously irregular group; pick the "thought" vowel, f when final. + out.push("ɔ".into()); + if rest_str == "ough" { + out.push("f".into()); + break; + } + if rest_str.starts_with("ought") { + out.push("t".into()); + i += 5; + continue; + } + i += 3; + continue; + } + if rest_str.starts_with("gu") && rest_str.len() > 2 { + // The u is silent before a vowel (guard, guess). + out.push("ɡ".into()); + i += 2; + continue; + } + if rest_str.starts_with("ps") && at_start { + out.push("s".into()); + i += 2; + continue; + } + let two: String = rest.iter().take(2).collect(); + let three: String = rest.iter().take(3).collect(); + match three.as_str() { + "igh" => { + push_vowel(&mut out, &mut vowel_positions, "I"); + i += 3; + continue; + } + "tch" => { + out.push("ʧ".into()); + i += 3; + continue; + } + "dge" => { + out.push("ʤ".into()); + i += 3; + continue; + } + _ => {} + } + match two.as_str() { + "ch" => { + out.push("ʧ".into()); + i += 2; + continue; + } + "sh" => { + out.push("ʃ".into()); + i += 2; + continue; + } + "th" => { + out.push("θ".into()); + i += 2; + continue; + } + "ph" => { + out.push("f".into()); + i += 2; + continue; + } + "ck" => { + out.push("k".into()); + i += 2; + continue; + } + "qu" => { + out.push("k".into()); + out.push("w".into()); + i += 2; + continue; + } + "wh" => { + out.push("w".into()); + i += 2; + continue; + } + "gh" => { + // Word-final → f (laugh, tough); before t → silent (knight's "igh" + // rule already handled it); otherwise g (ghost). + if i + 2 == n { + out.push("f".into()); + } else if chars.get(i + 2).copied() != Some('t') { + out.push("ɡ".into()); + } + i += 2; + continue; + } + "ng" if i + 2 == n => { + out.push("ŋ".into()); + i += 2; + continue; + } + "nk" => { + out.push("ŋ".into()); + out.push("k".into()); + i += 2; + continue; + } + "kn" if at_start => { + out.push("n".into()); + i += 2; + continue; + } + "gn" if at_start => { + out.push("n".into()); + i += 2; + continue; + } + "wr" if at_start => { + out.push("ɹ".into()); + i += 2; + continue; + } + "mb" if i + 2 == n => { + out.push("m".into()); + i += 2; + continue; + } + _ => {} + } + let Some(c) = chars.get(i) else { break }; + let c = *c; + if is_vowel_letter(c) { + // Vowel digraphs. + let digraph: String = rest.iter().take(2).collect(); + let digraph_symbol = match digraph.as_str() { + "ee" | "ea" | "ie" => Some("i"), + "oo" | "ue" | "ui" => Some("u"), + "ou" => Some("W"), + "ow" => Some("O"), + "oi" | "oy" => Some("Y"), + "ai" | "ay" | "ei" => Some("A"), + "au" | "aw" => Some("ɔ"), + "oa" | "oe" => Some("O"), + _ => None, + }; + if let Some(sym) = digraph_symbol.filter(|_| rest.len() >= 2) { + push_vowel(&mut out, &mut vowel_positions, sym); + i += 2; + continue; + } + // A doubled consonant after the vowel keeps it short (little → ˈlɪtəl). + let doubled = chars.get(i + 1) == chars.get(i + 2) && chars.get(i + 1).is_some_and(|c| !is_vowel_letter(*c)); + let long = long_last && last_vowel_index == Some(i) && !doubled; + let sym = match (c, long) { + ('a', true) => "A", + ('e', true) => "i", + ('i', true) => "I", + ('o', true) => "O", + ('u', true) => "u", + ('a', _) => "æ", + ('e', _) => "ɛ", + ('i', _) => "ɪ", + ('o', _) => "ɑ", + ('u', _) => "ʌ", + ('y', _) => "ɪ", + _ => "ə", + }; + push_vowel(&mut out, &mut vowel_positions, sym); + i += 1; + continue; + } + // Consonants. + match c { + 'c' => { + let soft = chars.get(i + 1).is_some_and(|n| matches!(n, 'e' | 'i' | 'y')); + out.push(if soft { "s".into() } else { "k".into() }); + } + 'g' => { + let soft = chars.get(i + 1).is_some_and(|n| matches!(n, 'e' | 'i' | 'y')); + out.push(if soft { "ʤ".into() } else { "ɡ".into() }); + } + 'x' => { + out.push("k".into()); + out.push("s".into()); + } + 'j' => out.push("ʤ".into()), + 'q' => { + out.push("k".into()); + } + 'l' if i + 2 == n && chars.get(i + 1) == Some(&'e') && i > 0 && chars.get(i - 1).is_some_and(|c| !is_vowel_letter(*c)) => { + // "little", "castle": the final "le" is a syllable əl. + out.push("ə".into()); + out.push("l".into()); + i += 2; + continue; + } + 'y' if at_start => out.push("j".into()), + 'y' if i + 1 == n => out.push("i".into()), + 'y' => out.push("ɪ".into()), + other => { + let sym = match other { + 'b' => "b", + 'd' => "d", + 'f' => "f", + 'h' => "h", + 'k' => "k", + 'l' => "l", + 'm' => "m", + 'n' => "n", + 'p' => "p", + 'r' => "ɹ", + 's' => "s", + 't' => "t", + 'v' => "v", + 'w' => "w", + 'z' => "z", + _ => "", + }; + if !sym.is_empty() { + out.push(sym.into()); + } + } + } + i += 1; + } + // Doubled consonants collapse ("narration": rr → r). + let mut collapsed: Vec = Vec::new(); + for sym in out { + if collapsed.last() == Some(&sym) && sym.chars().count() == 1 && !is_vowel_kokoro(&sym) { + continue; + } + collapsed.push(sym); + } + if collapsed.is_empty() { + return None; + } + // Primary stress on the first vowel if not already marked. + if !collapsed.iter().any(|s| s.contains('ˈ') || s.contains('ˌ')) + && let Some(pos) = collapsed.iter().position(|s| is_vowel_kokoro(s)) + && let Some(v) = collapsed.get_mut(pos) + { + *v = format!("ˈ{v}"); + } + Some(collapsed.join("")) +} + +fn push_vowel(out: &mut Vec, positions: &mut Vec, sym: &str) { + positions.push(out.len()); + out.push(sym.to_string()); +} + +fn is_vowel_letter(c: char) -> bool { + matches!(c, 'a' | 'e' | 'i' | 'o' | 'u' | 'y') +} + +fn is_vowel_kokoro(s: &str) -> bool { + let mut chars = s.chars(); + match chars.next() { + Some('ˈ') | Some('ˌ') => { + matches!(chars.next(), Some('ɑ' | 'æ' | 'ə' | 'ʌ' | 'ɔ' | 'ɛ' | 'ɚ' | 'ɜ' | 'ɪ' | 'ʊ' | 'i' | 'u' | 'A' | 'I' | 'O' | 'W' | 'Y')) + } + Some(c) => matches!(c, 'ɑ' | 'æ' | 'ə' | 'ʌ' | 'ɔ' | 'ɛ' | 'ɚ' | 'ɜ' | 'ɪ' | 'ʊ' | 'i' | 'u' | 'A' | 'I' | 'O' | 'W' | 'Y'), + None => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A small CMUdict-format dictionary typed for these tests (55 entries, + /// 53 distinct words). + const TEST_DICT: &str = r#";;; small test dictionary +HELLO HH AH0 L OW1 +WORLD W ER1 L D +READ R IY1 D +READ(2) R EH D +THE DH AH0 +OF AH0 V +TO T UW1 +AND AE1 N D +IN IH0 N +IS IH0 Z +IT IH0 T +FOR F AO1 R +ON AA1 N +WITH W IH0 TH +AT AE1 T +BY B AY1 +AS AE1 Z +BE B IY1 +OR AO1 R +BUT B AH1 T +FROM F R AH0 M +THAT DH AE1 T +THIS DH IH0 S +AN AE1 N +A AH0 +A(2) EY1 +I AY1 +T T IY1 +D D IY1 +CAT K AE1 T +DOG D AO1 G +WISH W IH1 SH +PLAY P L EY1 +MAKE M EY1 K +SLOW S L OW1 +DO D UW1 +FIVE F AY1 V +CENT S EH1 N T +THREE TH R IY1 +DOLLAR D AA1 L ER0 +FIFTY F IH1 F T IY0 +ZERO Z IH1 R OW +ONE W AH1 N +FOUR F AO1 R +THIRTY T ER1 T IY0 +SEVEN S EH1 V AH0 N +TWENTY T W EH1 N T IY0 +NINETY N AY1 N T IY0 +NINETEEN N AY1 N T IY1 N +TWO T UW1 +NINE N AY1 N +OCTOBER AA0 K T OW1 B ER0 +CALL K AO1 L +OFF AO1 F +PERCENT P ER0 S EH1 N T +"#; + + pub(crate) fn test_lex() -> Lexicon { + Lexicon::parse_cmudict(TEST_DICT.as_bytes()).unwrap() + } + + /// Flatten chunks: phonemes as-is, pauses as `[pause N]`. + fn flat(chunks: &[Chunk]) -> String { + let mut s = String::new(); + for c in chunks { + match c { + Chunk::Phonemes(p) => { + if !s.is_empty() { + s.push(' '); + } + s.push_str(p); + } + Chunk::Pause { millis } => { + if !s.is_empty() { + s.push(' '); + } + s.push_str(&format!("[pause {millis}]")); + } + } + } + s + } + + fn speak(text: &str) -> String { + flat(&phonemize(text, Lang::EnUs, &test_lex(), &Overrides::default()).unwrap()) + } + + #[test] + fn parses_cmudict() { + let lex = test_lex(); + assert_eq!(lex.len(), 53); + assert_eq!(lex.skipped(), 0); + // Alternates fold into one key; lookup uses the first pronunciation. + assert_eq!(lex.pronunciations("READ").map(Vec::as_slice), Some(&["R IY1 D".to_string(), "R EH D".to_string()][..])); + assert_eq!(lex.pronunciations("A").map(|v| v.len()), Some(2)); + assert!(lex.pronunciations("MISSING").is_none()); + assert!(Lexicon::parse_cmudict(b"").unwrap().is_empty()); + } + + #[test] + fn skips_malformed_lines_and_counts_them() { + let input = + b";;; a comment\n\nGOOD AE1\nBADLINE\nONLYWORD\nWORD XX1 YY2\nBAD( AE1\nWORD(2) AA1\nWORD AE1 # trailing comment\nTRAIL AE1 T # note\n"; + let lex = Lexicon::parse_cmudict(input).unwrap(); + assert_eq!(lex.len(), 3); // GOOD, WORD (two pronunciations), TRAIL + assert_eq!(lex.skipped(), 4); // BADLINE, ONLYWORD, WORD XX1 YY2, BAD( + assert_eq!(lex.pronunciations("WORD").map(|v| v.first().map(String::as_str)), Some(Some("AA1"))); + // The trailing comment is stripped, not part of the pronunciation. + assert_eq!(lex.pronunciations("TRAIL").map(|v| v.first().map(String::as_str)), Some(Some("AE1 T"))); + } + + #[test] + fn refuses_oversized_dictionaries() { + let oversized = vec![b'a'; MAX_DICT_BYTES + 1]; + assert!(matches!(Lexicon::parse_cmudict(&oversized), Err(LexError::TooLarge))); + let mut big = String::new(); + for i in 0..MAX_DICT_ENTRIES + 1 { + big.push_str(&format!("W{i} AE1\n")); + } + let big = big.into_bytes(); + assert!(matches!(Lexicon::parse_cmudict(&big), Err(LexError::TooManyEntries))); + } + + #[test] + fn table_cases() { + let cases: [(&str, &str); 46] = [ + // Dictionary words, stress placement, function words. + ("Hello world", "həlˈO wˈɜɹld"), + ("hello", "həlˈO"), + ("the world", "ðə wˈɜɹld"), + ("world", "wˈɜɹld"), + ("read", "ɹˈid"), + ("I read", "ˈI ɹˈid"), + ("T D A", "tˈi dˈi ˈA"), + ("NASA", "nˈæsæ"), + ("a cat", "ə kˈæt"), + ("an apple", "æn ˈæpəl"), + ("It is on", "ɪt ɪz ɑn"), + // Numbers from the normalizer. + ("$3.50", "θɹˈi dˈɑlɚz ænd fˈɪfti sˈɛnts"), + ("in 1999.", "ɪn nˈIntˈin nˈInti nˈIn."), + ("5 cats", "fˈIv kˈæts"), + ("Call 555-0134.", "kˈɔl fˈIv fˈIv fˈIv zˈɪɹO wˈʌn θɹˈi fˈɔɹ."), + ("At 3:30.", "æt θɹˈi tˈɜɹti."), + ("Up 20%.", "ˈʌp twˈɛnti pɚsˈɛnt."), + ("On 10/7/2026.", "ɑn ɑktˈObɚ sˈɛvɛnθ twˈɛnti twˈɛnti sˈɪks."), + // Morphology on dictionary stems. + ("worlds", "wˈɜɹldz"), + ("hellos", "həlˈOz"), + ("read's", "ɹˈidz"), + ("cats", "kˈæts"), + ("dogs", "dˈɔɡz"), + ("wishes", "wˈɪʃɪz"), + ("played", "plˈAd"), + ("making", "mˈAkɪŋ"), + ("slower", "slˈOɚ"), + ("slowly", "slˈOli"), + ("undo", "əndˈu"), + ("redo", "ɹidˈu"), + ("cents", "sˈɛnts"), + ("fives", "fˈIvz"), + // Letter-to-sound for unknown words. + ("kokoro", "kˈɑkɑɹɑ"), + ("filmcraft", "fˈɪlmkɹæft"), + ("knight", "nˈIt"), + // Punctuation attachment, kept and dropped. + ("hello, world.", "həlˈO, wˈɜɹld."), + ("hello $ world", "həlˈO wˈɜɹld"), + ("end.", "ˈɛnd."), + ("Wow!", "wˈO!"), + ("Really?", "ɹˈilɪ?"), + ("a: b; c", "ə: bˈi; sˈi"), + ("Tom & Jerry", "tˈɑm ænd ʤˈɛɹɪ"), + ("Version 2. Next.", "vˈɛɹʒən tˈu. nˈɛkst."), + ("It was 3.5 percent.", "ɪt wˈæs θɹˈi pˈYnt fˈIv pɚsˈɛnt."), + // The review's two sentences, end to end. + ("Dr. Smith lives on Main St. in 1999.", "dˈɑktɑɹ smˈɪθ lˈɪvɛs ɑn mˈAn stɹˈit ɪn nˈIntˈin nˈInti nˈIn."), + ( + "The TDA project for NASA costs $1,234,567.", + "ðə tˈi dˈi ˈA pɹˈɑʤɛkt fɔɹ nˈæsæ kˈɑsts wˈʌn mˈɪlɪɑn tˈu hˈʌndɹɛd tˈɜɹti fˈɔɹ θˈWsænd fˈIv hˈʌndɹɛd sˈɪkstɪ sˈɛvən dˈɑlɚz.", + ), + ]; + for (input, expected) in &cases { + assert_eq!(&speak(input), expected, "input {input:?}"); + } + } + + #[test] + fn overrides_both_forms_and_caps() { + // Raw Kokoro phonemes and a respelling. + let ov = Overrides([("kokoro".into(), "/kˈOkOɹO/".into()), ("tda".into(), "tee dee ay".into())].into_iter().collect()); + assert_eq!(speak_with(&ov, "kokoro"), "kˈOkOɹO"); + assert_eq!(speak_with(&ov, "TDA"), "tˈi dˈi ˈA"); + // Case-insensitive keys. + assert_eq!(speak_with(&ov, "KOKORO"), "kˈOkOɹO"); + // A raw value with a character outside SYMBOLS falls back to respelling. + let bad = Overrides([("kokoro".into(), "/kəmˈpuːtər/".into())].into_iter().collect()); + assert_eq!(speak_with(&bad, "kokoro"), "kˈɑkɑɹɑ"); + } + + fn speak_with(ov: &Overrides, text: &str) -> String { + flat(&phonemize(text, Lang::EnUs, &test_lex(), ov).unwrap()) + } + + #[test] + fn letter_to_sound_cases() { + let cases: [(&str, &str); 36] = [ + ("filmcraft", "fˈɪlmkɹæft"), + ("narration", "næɹˈæʃən"), + ("kokoro", "kˈɑkɑɹɑ"), + ("phone", "fˈOn"), + ("knight", "nˈIt"), + ("cake", "kˈAk"), + ("quickly", "kwˈɪklɪ"), + ("thought", "θˈɔt"), + ("ghost", "ɡˈɑst"), + ("chemistry", "ʧˈɛmɪstɹɪ"), + ("giant", "ʤˈɪænt"), + ("gym", "ʤˈɪm"), + ("xylophone", "ksˈɪlɑfOn"), + ("little", "lˈɪtəl"), + ("ocean", "ˈɑsin"), + ("python", "pˈɪθɑn"), + ("rhythm", "ɹhˈɪθm"), + ("island", "ˈɪslænd"), + ("listen", "lˈɪsən"), + ("castle", "kˈæstəl"), + ("answer", "ˈænswɛɹ"), + ("autumn", "ˈɔtʌmn"), + ("conscious", "kˈɑnsɪWs"), + ("schedule", "skˈɛdul"), + ("nature", "nˈæʧɚ"), + ("measure", "mˈiʒɚ"), + ("vision", "vˈɪʒən"), + ("education", "ɛdʌkˈæʃən"), + ("example", "ˈɛksæmpəl"), + ("exhibit", "ˈɛkshɪbɪt"), + ("hour", "hˈWɹ"), + ("honest", "hˈɑnɛst"), + ("wrong", "ɹˈɑŋ"), + ("wren", "ɹˈɛn"), + ("gnome", "nˈOm"), + ("psalm", "sˈælm"), + ]; + for (word, expected) in &cases { + assert_eq!(&letter_to_sound(word).unwrap(), expected, "word {word}"); + } + // More, checked for pronounceability and non-emptiness rather than exact form. + for w in ["lamb", "climb", "song", "bank", "laugh", "tough", "cough", "plague", "guard", "business", "colonel", "smile", "bridge", "wrist", "knock"] { + let out = letter_to_sound(w).unwrap(); + assert!(!out.is_empty(), "{w} produced nothing"); + assert!(out.chars().all(|c| SYMBOLS.contains(c)), "{w} → {out:?}"); + println!("LTS {w} -> {out}"); + } + } + + #[test] + fn chunks_are_bounded_and_split_at_punctuation() { + // A long sentence with many commas: splits at commas, each chunk ≤ 400. + let sentence = "one, two, three, ".repeat(200); + let chunks = phonemize(&sentence, Lang::EnUs, &test_lex(), &Overrides::default()).unwrap(); + assert!(chunks.len() > 1, "expected a split, got {} chunk(s)", chunks.len()); + for c in &chunks { + let Chunk::Phonemes(p) = c else { panic!("not phonemes") }; + assert!(p.chars().count() <= MAX_CHUNK_CHARS, "chunk of {} chars", p.chars().count()); + } + // The split happened at commas: no chunk ends mid-word. + for c in &chunks[..chunks.len() - 1] { + let Chunk::Phonemes(p) = c else { continue }; + assert!( + p.ends_with(',') || p.ends_with(';') || p.ends_with(':') || p.ends_with('w') || p.ends_with("ʌn") || p.ends_with("tˈu"), + "bad split: {p:?}" + ); + } + } + + #[test] + fn every_output_char_is_in_symbols() { + let inputs = [ + "Call 555-0134 by 3:30 p.m. on Oct. 7th, it's $3.50 (20% off)", + "Hello world. [pause 1s] T D A & NASA, really?!", + "1,234,567 dollars; 1999 was … fun — (honest)!", + "Version 2. Next. Up 20%. At 3:30. On 10/7/2026.", + "", + " ", + "$$$$", + "[pause", + "[pause x]", + "日本語", + "emoji 👍👍👍", + "العربية", + "\u{0301}", + "\0", + "3:", + "1,,2", + ".....", + "[[[[", + "-", + "$", + "3:305", + "5-", + "999999999999999999999999", + "007", + "12345678901234", + "filmcraft narration kokoro", + "Dr. Smith lives on Main St. in 1999.", + "The TDA project for NASA costs $1,234,567.", + ]; + for input in &inputs { + let Ok(chunks) = phonemize(input, Lang::EnUs, &test_lex(), &Overrides::default()) else { continue }; + for c in chunks { + let Chunk::Phonemes(p) = c else { continue }; + for ch in p.chars() { + assert!(SYMBOLS.contains(ch), "{input:?} produced {p:?} — bad char {ch:?} U+{:04X}", ch as u32); + } + } + } + } + + #[test] + fn hostile_inputs_do_not_panic() { + let ov_big = Overrides((0..1001).map(|i| (format!("w{i}"), "/ɑ/".to_string())).collect()); + let ov_long = Overrides([("x".into(), "a".repeat(201))].into_iter().collect()); + let huge = "a".repeat(2 * 1024 * 1024); + let long_word = "a".repeat(10_000); + let default = Overrides::default(); + let hostile: Vec<(&str, &Overrides)> = vec![ + ("", &default), + (" \t\n ", &default), + (huge.as_str(), &default), + (long_word.as_str(), &default), + ("$$$$", &default), + ("日本語のテキスト", &default), + ("emoji 👍👍👍", &default), + ("العربية", &default), + ("\u{0301}", &default), + ("\0", &default), + ("hello", &ov_big), + ("hello", &ov_long), + ("[pause 999999999999s]", &default), + ]; + for (input, ov) in &hostile { + let result = std::panic::catch_unwind(|| phonemize(input, Lang::EnUs, &test_lex(), ov)); + assert!(result.is_ok(), "panicked on {input:?}"); + } + // The oversized input errors instead of degrading. + assert!(phonemize(&huge, Lang::EnUs, &test_lex(), &default).is_err()); + } + + #[test] + fn overrides_are_capped_not_errors() { + let mut map = BTreeMap::new(); + for i in 0..1100 { + map.insert(format!("w{i}"), "/ɑ/".to_string()); + } + map.insert("hello".to_string(), "/oʊ/".to_string()); // beyond 1000? keys sorted: w0.. so "hello" is after w-entries → dropped + let ov = Overrides(map); + // 1001st+ entries ignored silently; no panic, no error. + let out = phonemize("hello", Lang::EnUs, &test_lex(), &ov).unwrap(); + assert!(!out.is_empty()); + } + + /// Measured accuracy against a real CMUdict, when the environment provides one + /// (FILMCRAFT_CMUDICT=/path/to/cmudict.dict). Holds out every 50th entry, + /// rebuilds the lexicon without them, and reports letter-to-sound word + /// accuracy and phoneme error rate on the held-out words. + #[test] + fn measured_accuracy_against_cmudict() { + let Ok(path) = std::env::var("FILMCRAFT_CMUDICT") else { + eprintln!("measured_accuracy_against_cmudict: FILMCRAFT_CMUDICT not set; skipping"); + return; + }; + let Ok(bytes) = std::fs::read(&path) else { + eprintln!("measured_accuracy_against_cmudict: cannot read {path}; skipping"); + return; + }; + let full = Lexicon::parse_cmudict(&bytes).unwrap(); + // Hold out every 50th entry. + let mut kept_text = String::new(); + let mut held_out: Vec<(String, String)> = Vec::new(); + for (i, (word, prons)) in full.entries.iter().enumerate() { + if i % 50 == 49 { + if let Some(first) = prons.first() { + held_out.push((word.clone(), first.clone())); + } + continue; + } + for pron in prons { + kept_text.push_str(&format!("{word} {pron}\n")); + } + } + let lex = Lexicon::parse_cmudict(kept_text.as_bytes()).unwrap(); + let _ = &lex; + let mut words_hit = 0usize; + let mut phoneme_total = 0usize; + let mut phoneme_errors = 0usize; + for (word, truth) in &held_out { + let Some(guess) = letter_to_sound(&word.to_lowercase()) else { continue }; + let expected = arpabet_to_kokoro(truth, false); + let g: Vec = guess.chars().collect(); + let e: Vec = expected.chars().collect(); + if g == e { + words_hit += 1; + } + phoneme_total += e.len().max(1); + phoneme_errors += edit_distance(&g, &e); + } + let n = held_out.len().max(1); + let accuracy = words_hit as f64 / n as f64; + let per = phoneme_errors as f64 / phoneme_total as f64; + println!("held out {} words: word accuracy {:.1}%, phoneme error rate {:.1}%", words_hit, accuracy * 100.0, per * 100.0); + assert!(!held_out.is_empty()); + } + + /// Levenshtein distance over phoneme characters. + fn edit_distance(a: &[char], b: &[char]) -> usize { + let mut prev: Vec = (0..=b.len()).collect(); + for i in 1..=a.len() { + let mut cur = vec![i; b.len() + 1]; + for j in 1..=b.len() { + let sub = prev[j - 1] + usize::from(a[i - 1] != b[j - 1]); + cur[j] = sub.min(prev[j] + 1).min(cur[j - 1] + 1); + } + prev = cur; + } + prev[b.len()] + } +} diff --git a/crates/tts/Cargo.toml b/crates/tts/Cargo.toml new file mode 100644 index 000000000..5100d70db --- /dev/null +++ b/crates/tts/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "filmcraft-tts" +description = "FilmCraft text to speech: the Voice trait, the voice catalogue, script parsing (pause markers), the built-in formant voice and (feature kokoro) Kokoro-82M neural voices on candle" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[features] +default = [] +# Kokoro-82M neural voices on the CPU via candle (pure Rust). Off by default and in wasm builds. +kokoro = ["dep:candle-core", "dep:candle-nn", "dep:zip", "dep:serde_json", "dep:filmcraft-tts-text", "dep:filmcraft-audio-dsp"] + +[dependencies] +filmcraft-tts-text = { workspace = true, optional = true } +filmcraft-audio-dsp = { workspace = true, optional = true } +serde = { workspace = true } +thiserror = { workspace = true } +candle-core = { version = "0.9", default-features = false, optional = true } +candle-nn = { version = "0.9", default-features = false, optional = true } +zip = { version = "7", default-features = false, optional = true } +serde_json = { workspace = true, optional = true } + +[[example]] +name = "kokoro_say" +required-features = ["kokoro"] + +[[example]] +name = "narrate" +required-features = ["kokoro"] diff --git a/crates/tts/README.md b/crates/tts/README.md new file mode 100644 index 000000000..a7993f92e --- /dev/null +++ b/crates/tts/README.md @@ -0,0 +1,83 @@ +# filmcraft-tts + +Text to speech for FilmCraft's Text to Speech panel (narrations, L2). Engine commands (`tts.*`) and +clip behaviour: `crates/engine/src/narration.rs`. + +- `Voice` trait (script in, mono 24 kHz `f32` out), the voice catalogue (`voices`, `voice`), and + the limits (64 KB script, 30-minute narration, pace 0.5–2×, pitch ±12 semitones). +- `script`: pause markers `[pause 1s]` / `[pause 500ms]` (case-insensitive, capped at 10 s; + malformed markers are read as text). +- `formant`: the built-in voices **Basic Female** and **Basic Male**, an original source–filter + formant synthesizer: spelling rules → phone targets → glottal pulse train through three two-pole + resonators, plus filtered noise for fricatives and bursts. No download, works in the web build. + +- `kokoro` (feature `kokoro`): **Kokoro-82M neural voices** in pure Rust on candle (CPU): ALBERT + text encoder, StyleTTS 2 prosody predictor (durations, F0, energy) and decoder, iSTFTNet + vocoder with a harmonic-plus-noise source. Weights are read from the official PyTorch + checkpoint with candle's restricted pickle reader (no code from the file runs); voice packs from + their zip archive. `config.json` is checked against the architecture this code implements. +- `neural` (feature `kokoro`): the natural voices as `Voice`s: script → `filmcraft-tts-text` + (normalizer, CMUdict + letter-to-sound, ≤ 400-symbol chunks, pauses) → Kokoro per chunk → exact + silence for pauses, 120 ms between sentences → vocal pitch with `filmcraft-audio-dsp`'s pitch + shifter (Kokoro has no pitch input). Pace is Kokoro's speed. +- `catalog`: the natural-voice package (Kokoro-82M, 9 US English voices graded C+ or better on the + model card, CMUdict; 336 MB), every file pinned to a revision and SHA-256. Downloaded by the + engine (`tts.downloadVoices`) after the user confirms; never bundled or committed. + +Everything is deterministic (the vocoder's noise is seeded). Without features the crate depends +only on `serde` and `thiserror` (and checks for wasm). + +## Measured quality and speed (M7.12, 2026-10-08) + +50 original sentences (`tests/data/sentences.txt`: numbers, money, times, dates, phone numbers, +acronyms, names) spoken by each voice through the full pipeline, transcribed by Whisper-base +(FilmCraft's own, `filmcraft-speech`), word error rate after lowercasing and removing punctuation +(number formatting such as "six" vs "6" still counts as an error): + +| Voice | WER | | Voice | WER | +|---|---|---|---|---| +| Heart | 2.1 % | | Puck | 2.8 % | +| Aoede | 2.4 % | | Michael | 3.2 % | +| Nicole | 2.8 % | | Kore | 3.4 % | +| Bella | 3.0 % | | Fenrir, Sarah | 3.6 % | +| Basic Female (built in) | 96.6 % | | | | + +Speed: 2.2–2.3× real time on one desktop CPU (Linux x86-64, load average ≈ 15 from parallel +builds), e.g. 186 s of speech in 84 s; model load 0.5 s. Reproduce with the `narrate` example and +`filmcraft-speech`'s `transcribe` example. + +## Limitations + +- The built-in voices are **robotic** and nearly unintelligible to a recogniser (above); they are + the no-download fallback and the test voice. +- English (United States) only. Words outside CMUdict go through letter-to-sound rules and are + often mispronounced (see `crates/tts-text/README.md`); homographs take the first pronunciation. +- About 2× faster than real time on the CPU: a one-minute narration takes about 30 s. The UI runs + synthesis as a background job. +- Kokoro's model card states the weights are Apache-2.0 but the repository has no LICENSE file. +- Vocal pitch shifts the voice's base pitch (and its formants slightly); pace scales every duration. + +## Tests + +`cargo test -p filmcraft-tts`: determinism, valid samples (finite, peak ≤ 0.8, audible RMS), pause +markers produce exactly the requested number of zero samples, pace 2× / 0.5× gives 0.5× / 2× the +length (±5 % / ±10 %), measured pitch (autocorrelation) of the female voice is ≥ 1.5× the male, and ++6 semitones raises it by √2 (±0.12); empty, emoji-only, non-Latin, pause-only and over-long scripts +are refused before rendering; hostile scripts never panic. + +Model-dependent tests run when `FILMCRAFT_KOKORO_DIR` names a downloaded package directory +(determinism, whole frames, pace changes length, refusals); the loaders have hostile-input tests +(truncated or oversized voice data, wrong byte order, NaN, non-zip files, mismatched config). + +## References + +- G. E. Peterson and H. L. Barney, "Control Methods Used in a Study of the Vowels", JASA 24 (1952): + the published average formant frequencies used for the vowels. +- D. H. Klatt, "Software for a cascade/parallel formant synthesizer", JASA 67 (1980): the standard + two-pole digital resonator form. No code was read or copied; everything else is original. +- Kokoro (implemented from papers, `config.json` and the checkpoint's tensor names and shapes; no + reference code read): Y. A. Li et al., "StyleTTS 2" (NeurIPS 2023); T. Kaneko et al., "iSTFTNet" + (ICASSP 2022); Z. Lan et al., "ALBERT" (ICLR 2020); J. Kong et al., "HiFi-GAN" (NeurIPS 2020); + X. Wang et al., "Neural source-filter waveform models" (2019); L. Ziyin et al., "Neural Networks + Fail to Learn Periodic Functions and How to Fix It" (Snake, NeurIPS 2020). Weights: hexgrad, + Kokoro-82M, Apache-2.0. diff --git a/crates/tts/examples/kokoro_say.rs b/crates/tts/examples/kokoro_say.rs new file mode 100644 index 000000000..e3b577b21 --- /dev/null +++ b/crates/tts/examples/kokoro_say.rs @@ -0,0 +1,46 @@ +//! `cargo run -p filmcraft-tts --release --features kokoro --example kokoro_say -- "" out.wav [speed]` +//! +//! Speaks Kokoro phonemes with a voice pack and writes a 24 kHz mono 32-bit float WAV. + +fn main() -> Result<(), Box> { + let a: Vec = std::env::args().collect(); + let (Some(dir), Some(voice), Some(ph), Some(out)) = (a.get(1), a.get(2), a.get(3), a.get(4)) else { + eprintln!("usage: kokoro_say [speed]"); + std::process::exit(2); + }; + let speed: f64 = a.get(5).and_then(|s| s.parse().ok()).unwrap_or(1.0); + let t0 = std::time::Instant::now(); + let k = filmcraft_tts::kokoro::Kokoro::load(std::path::Path::new(dir))?; + let v = filmcraft_tts::kokoro::VoicePack::load(std::path::Path::new(voice))?; + let t1 = std::time::Instant::now(); + let s = k.speak(ph, &v, speed, 1)?; + let t2 = std::time::Instant::now(); + let rate = filmcraft_tts::kokoro::KOKORO_RATE; + let mut w = Vec::with_capacity(44 + s.len() * 4); + let data = (s.len() * 4) as u32; + w.extend_from_slice(b"RIFF"); + w.extend_from_slice(&(36 + data).to_le_bytes()); + w.extend_from_slice(b"WAVEfmt "); + w.extend_from_slice(&16u32.to_le_bytes()); + w.extend_from_slice(&3u16.to_le_bytes()); + w.extend_from_slice(&1u16.to_le_bytes()); + w.extend_from_slice(&rate.to_le_bytes()); + w.extend_from_slice(&(rate * 4).to_le_bytes()); + w.extend_from_slice(&4u16.to_le_bytes()); + w.extend_from_slice(&32u16.to_le_bytes()); + w.extend_from_slice(b"data"); + w.extend_from_slice(&data.to_le_bytes()); + for x in &s { + w.extend_from_slice(&x.to_le_bytes()); + } + std::fs::write(out, w)?; + let secs = s.len() as f64 / f64::from(rate); + eprintln!( + "load {:.2}s, synth {:.2}s for {secs:.2}s of audio (×{:.1} real time), peak {:.3}", + (t1 - t0).as_secs_f64(), + (t2 - t1).as_secs_f64(), + secs / (t2 - t1).as_secs_f64(), + s.iter().fold(0f32, |m, x| m.max(x.abs())) + ); + Ok(()) +} diff --git a/crates/tts/examples/narrate.rs b/crates/tts/examples/narrate.rs new file mode 100644 index 000000000..0408d0cd0 --- /dev/null +++ b/crates/tts/examples/narrate.rs @@ -0,0 +1,49 @@ +//! `cargo run -p filmcraft-tts --release --features kokoro --example narrate -- ` +//! +//! Speaks a script file with any FilmCraft voice (built in or natural) through the full pipeline +//! (normalizer, pronunciation, Kokoro) and writes a mono 32-bit float WAV. + +fn main() -> Result<(), Box> { + let a: Vec = std::env::args().collect(); + let (Some(models), Some(id), Some(script), Some(out)) = (a.get(1), a.get(2), a.get(3), a.get(4)) else { + eprintln!("usage: narrate "); + std::process::exit(2); + }; + let text = std::fs::read_to_string(script)?; + let t0 = std::time::Instant::now(); + let v = filmcraft_tts::voice_in(id, Some(std::path::Path::new(models)))?; + let t1 = std::time::Instant::now(); + let audio = v.synthesize(&text, &filmcraft_tts::Params::default())?; + let t2 = std::time::Instant::now(); + let rate = audio.sample_rate; + let data = (audio.samples.len() * 4) as u32; + let mut w = Vec::with_capacity(44 + data as usize); + for part in [ + &b"RIFF"[..], + &(36 + data).to_le_bytes(), + b"WAVEfmt ", + &16u32.to_le_bytes(), + &3u16.to_le_bytes(), + &1u16.to_le_bytes(), + &rate.to_le_bytes(), + &(rate * 4).to_le_bytes(), + &4u16.to_le_bytes(), + &32u16.to_le_bytes(), + b"data", + &data.to_le_bytes(), + ] { + w.extend_from_slice(part); + } + for x in &audio.samples { + w.extend_from_slice(&x.to_le_bytes()); + } + std::fs::write(out, w)?; + eprintln!( + "{id}: load {:.2}s, synth {:.2}s for {:.2}s of audio (×{:.1} real time)", + (t1 - t0).as_secs_f64(), + (t2 - t1).as_secs_f64(), + audio.seconds(), + audio.seconds() / (t2 - t1).as_secs_f64() + ); + Ok(()) +} diff --git a/crates/tts/src/catalog.rs b/crates/tts/src/catalog.rs new file mode 100644 index 000000000..a74ff07d9 --- /dev/null +++ b/crates/tts/src/catalog.rs @@ -0,0 +1,154 @@ +//! The downloadable neural voices: Kokoro-82M, its voice packs and the CMUdict pronunciation +//! dictionary. Every file is pinned to a revision and checked against its SHA-256 when downloaded +//! (the downloader is `filmcraft_speech::models::download_files`, run by the engine after the +//! user confirms). Nothing here is bundled with FilmCraft or committed to the repository. +//! +//! Licences: Kokoro-82M weights and voices by hexgrad, Apache-2.0 (declared on the model card; the +//! repository has no separate LICENSE file). CMUdict, Carnegie Mellon University, BSD-style. +//! Pins verified 2026-10-07 against the Hugging Face API and by `sha256sum`. + +use std::path::{Path, PathBuf}; + +use serde::Serialize; + +use crate::{Gender, VoiceInfo}; + +/// One file of the neural voice package. +#[derive(Clone, Copy, Debug, Serialize)] +pub struct ModelFile { + /// Local file name inside the package directory. + pub name: &'static str, + pub url: &'static str, + pub sha256: &'static str, + pub size: u64, +} + +/// Directory name of the package under the models directory. +pub const PACKAGE_ID: &str = "kokoro-82m"; +pub const PACKAGE_NAME: &str = "Natural voices (Kokoro-82M)"; +pub const LICENSE: &str = "Apache-2.0 (Kokoro-82M by hexgrad); CMUdict: BSD-style (Carnegie Mellon University)"; +pub const SOURCE: &str = "https://huggingface.co/hexgrad/Kokoro-82M"; + +macro_rules! hf { + ($name:literal, $path:literal, $sha:literal, $size:expr) => { + ModelFile { + name: $name, + url: concat!("https://huggingface.co/hexgrad/Kokoro-82M/resolve/f3ff3571791e39611d31c381e3a41a3af07b4987/", $path), + sha256: $sha, + size: $size, + } + }; +} + +pub static FILES: &[ModelFile] = &[ + hf!("config.json", "config.json", "5abb01e2403b072bf03d04fde160443e209d7a0dad49a423be15196b9b43c17f", 2351), + hf!("kokoro-v1_0.pth", "kokoro-v1_0.pth", "496dba118d1a58f5f3db2efc88dbdc216e0483fc89fe6e47ee1f2c53f18ad1e4", 327_212_226), + ModelFile { + name: "cmudict.dict", + url: "https://raw.githubusercontent.com/cmusphinx/cmudict/74790861f652b15e4ac49015a90074ad62a27690/cmudict.dict", + sha256: "81917843c7f44ce2b094ac63873c2c7a4cf802040792c455ba3ca406891c3d22", + size: 3_618_488, + }, + hf!("af_heart.pt", "voices/af_heart.pt", "0ab5709b8ffab19bfd849cd11d98f75b60af7733253ad0d67b12382a102cb4ff", 523_425), + hf!("af_bella.pt", "voices/af_bella.pt", "8cb64e02fcc8de0327a8e13817e49c76c945ecf0052ceac97d3081480e8e48d6", 523_425), + hf!("af_nicole.pt", "voices/af_nicole.pt", "c5561808bcf5250fe8c5f5de32caf2d94f27e57e95befdb098c5c85991d4c5da", 523_430), + hf!("af_aoede.pt", "voices/af_aoede.pt", "c03bd1a4c3716c2d8eaa3d50022f62d5c31cfbd6e15933a00b17fefe13841cc4", 523_425), + hf!("af_kore.pt", "voices/af_kore.pt", "8bfbc512321c3db49dff984ac675fa5ac7eaed5a96cc31104d3a9080e179d69d", 523_420), + hf!("af_sarah.pt", "voices/af_sarah.pt", "49bd364ea3be9eb3e9685e8f9a15448c4883112a7c0ff7ab139fa4088b08cef9", 523_425), + hf!("am_michael.pt", "voices/am_michael.pt", "9a443b79a4b22489a5b0ab7c651a0bcd1a30bef675c28333f06971abbd47bd37", 523_435), + hf!("am_fenrir.pt", "voices/am_fenrir.pt", "98e507eca1db08230ae3b6232d59c10aec9630022d19accac4f5d12fcec3c37a", 523_430), + hf!("am_puck.pt", "voices/am_puck.pt", "dd1d8973f4ce4b7d8ae407c77a435f485dabc052081b80ea75c4f30b84f36223", 523_420), +]; + +/// A neural voice: catalogue entry and its voice-pack file. +#[derive(Clone, Copy, Debug)] +pub struct NeuralVoice { + pub info: VoiceInfo, + pub file: &'static str, +} + +const NEURAL_LICENSE: &str = "Apache-2.0 (Kokoro-82M by hexgrad)"; + +macro_rules! voice { + ($id:literal, $name:literal, $gender:ident, $file:literal, $desc:literal) => { + NeuralVoice { + info: VoiceInfo { + id: $id, + name: $name, + language: "en-US", + gender: Gender::$gender, + engine: "neural", + description: $desc, + license: NEURAL_LICENSE, + installed: false, + }, + file: $file, + } + }; +} + +/// US English voices the model card grades C+ or better, best first. +pub static VOICES: &[NeuralVoice] = &[ + voice!("kokoro-heart", "Heart", Female, "af_heart.pt", "Natural voice, warm (model card grade A)."), + voice!("kokoro-bella", "Bella", Female, "af_bella.pt", "Natural voice, bright (grade A−)."), + voice!("kokoro-nicole", "Nicole", Female, "af_nicole.pt", "Natural voice, soft (grade B−)."), + voice!("kokoro-michael", "Michael", Male, "am_michael.pt", "Natural voice (grade C+)."), + voice!("kokoro-fenrir", "Fenrir", Male, "am_fenrir.pt", "Natural voice, deep (grade C+)."), + voice!("kokoro-puck", "Puck", Male, "am_puck.pt", "Natural voice, lively (grade C+)."), + voice!("kokoro-aoede", "Aoede", Female, "af_aoede.pt", "Natural voice (grade C+)."), + voice!("kokoro-kore", "Kore", Female, "af_kore.pt", "Natural voice (grade C+)."), + voice!("kokoro-sarah", "Sarah", Female, "af_sarah.pt", "Natural voice (grade C+)."), +]; + +pub fn find(id: &str) -> Option<&'static NeuralVoice> { + VOICES.iter().find(|v| v.info.id == id) +} + +/// Total download size. +pub fn size() -> u64 { + FILES.iter().map(|f| f.size).sum() +} + +pub fn package_dir(models_dir: &Path) -> PathBuf { + models_dir.join(PACKAGE_ID) +} + +/// Every file present with its pinned size (contents were verified when downloaded). +pub fn installed(models_dir: &Path) -> bool { + let d = package_dir(models_dir); + FILES.iter().all(|f| std::fs::metadata(d.join(f.name)).is_ok_and(|m| m.len() == f.size)) +} + +/// Bytes still to download. +pub fn missing_bytes(models_dir: &Path) -> u64 { + let d = package_dir(models_dir); + FILES.iter().filter(|f| !std::fs::metadata(d.join(f.name)).is_ok_and(|m| m.len() == f.size)).map(|f| f.size).sum() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn files_are_pinned_and_voices_resolve() { + for f in FILES { + assert_eq!(f.sha256.len(), 64, "{}", f.name); + assert!(f.sha256.bytes().all(|b| b.is_ascii_hexdigit())); + assert!( + f.url.starts_with("https://") + && (f.url.contains("/f3ff3571791e39611d31c381e3a41a3af07b4987/") || f.url.contains("/74790861f652b15e4ac49015a90074ad62a27690/")), + "{}", + f.url + ); + assert!(!f.name.contains('/'), "flat package layout: {}", f.name); + } + for v in VOICES { + assert!(FILES.iter().any(|f| f.name == v.file), "{} has no file", v.info.id); + assert_eq!(find(v.info.id).map(|x| x.file), Some(v.file)); + } + assert!(size() > 330_000_000 && size() < 340_000_000, "{}", size()); + let dir = std::env::temp_dir().join(format!("filmcraft-tts-cat-{}", std::process::id())); + assert!(!installed(&dir)); + assert_eq!(missing_bytes(&dir), size()); + } +} diff --git a/crates/tts/src/formant.rs b/crates/tts/src/formant.rs new file mode 100644 index 000000000..00c321d39 --- /dev/null +++ b/crates/tts/src/formant.rs @@ -0,0 +1,736 @@ +//! The built-in voices: an original source–filter formant synthesizer. +//! +//! Text is turned into phones by simple English spelling rules ([`phones`]): digraphs (`th`, `sh`, +//! `ee`, `oo`, `igh`…), a silent final `e` that lengthens the vowel before it (`make`, `time`), +//! and digits read as words. Each phone is a target (three formant frequencies and bandwidths, +//! voicing and noise amplitudes, a duration). The renderer moves between targets with short +//! transitions, drives a cascade of three two-pole resonators with a glottal pulse train for voiced +//! sounds, and a resonator at the phone's noise frequency with white noise for fricatives, bursts +//! and aspiration. Pitch follows a falling line across each sentence and rises at a question mark. +//! +//! Vowel formants are the published adult averages of Peterson & Barney, "Control Methods Used in +//! a Study of the Vowels" (JASA 24, 1952). The digital resonator is the standard two-pole form +//! described in D. H. Klatt, "Software for a cascade/parallel formant synthesizer" (JASA 67, 1980). +//! Everything else is original. The result is intelligible-ish and clearly robotic: it is the +//! no-download fallback, not the main voice. + +use crate::script::{self, Segment}; +use crate::{Audio, Gender, MAX_SECONDS, Params, SAMPLE_RATE, TtsError, Voice, VoiceInfo, check_text}; + +/// A built-in voice: catalogue entry plus its speaking parameters. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct VoiceSpec { + pub info: VoiceInfo, + /// Base pitch in Hz. + pub f0: f64, + /// Formant scale (vocal tract length): 1 = adult male averages. + pub formant_scale: f64, + /// Breath noise mixed into voicing, 0–1. + pub breath: f32, +} + +const LICENSE: &str = "Built into FilmCraft (project licence)"; + +pub static VOICES: &[VoiceSpec] = &[ + VoiceSpec { + info: VoiceInfo { + id: "basic-female", + name: "Basic Female", + language: "en-US", + gender: Gender::Female, + engine: "basic", + description: "Built-in voice. No download; robotic.", + license: LICENSE, + installed: true, + }, + f0: 205.0, + formant_scale: 1.17, + breath: 0.06, + }, + VoiceSpec { + info: VoiceInfo { + id: "basic-male", + name: "Basic Male", + language: "en-US", + gender: Gender::Male, + engine: "basic", + description: "Built-in voice. No download; robotic.", + license: LICENSE, + installed: true, + }, + f0: 112.0, + formant_scale: 1.0, + breath: 0.03, + }, +]; + +/// A built-in voice ready to speak. +pub struct FormantVoice { + spec: VoiceSpec, +} + +impl FormantVoice { + pub fn new(spec: VoiceSpec) -> Self { + FormantVoice { spec } + } +} + +impl Voice for FormantVoice { + fn info(&self) -> VoiceInfo { + self.spec.info + } + + fn synthesize(&self, text: &str, params: &Params) -> Result { + check_text(text)?; + params.check()?; + let segments = script::parse(text); + // phones per speech segment, and the length check before anything is rendered + let mut plan: Vec = Vec::with_capacity(segments.len()); + let mut total_ms = 0.0f64; + let mut spoken = false; + for seg in &segments { + match seg { + Segment::Pause { millis } => { + total_ms += f64::from(*millis); + plan.push(Piece::Pause(*millis)); + } + Segment::Speech(s) => { + let ph = phones(s); + spoken |= ph.iter().any(|p| p.kind != Kind::Silence); + total_ms += ph.iter().map(|p| f64::from(p.dur_ms)).sum::() / params.pace; + plan.push(Piece::Speech(ph)); + } + } + if total_ms > MAX_SECONDS * 1000.0 { + return Err(TtsError::AudioTooLong); + } + } + if !spoken { + return Err(TtsError::Empty); + } + let estimate = (total_ms / 1000.0 * f64::from(SAMPLE_RATE)).ceil() as usize; + let mut out: Vec = Vec::new(); + out.try_reserve(estimate.saturating_add(SAMPLE_RATE as usize)).map_err(|e| TtsError::Invalid(format!("unable to allocate the narration: {e}")))?; + for piece in &plan { + match piece { + Piece::Pause(ms) => { + let n = (f64::from(*ms) / 1000.0 * f64::from(SAMPLE_RATE)).round() as usize; + out.resize(out.len().saturating_add(n), 0.0); + } + Piece::Speech(ph) => render(&self.spec, ph, params, &mut out), + } + } + // one gain for the whole narration so loudness doesn't jump between pieces + let peak = out.iter().fold(0.0f32, |m, s| m.max(s.abs())); + if peak > 0.0 && peak.is_finite() { + let g = 0.8 / peak; + for s in &mut out { + *s *= g; + } + } else if !peak.is_finite() { + return Err(TtsError::Invalid("synthesis produced invalid samples".into())); + } + Ok(Audio { samples: out, sample_rate: SAMPLE_RATE }) + } +} + +enum Piece { + Pause(u32), + Speech(Vec), +} + +// ------------------------------------------------------------------------------------- phones + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Kind { + Vowel, + Glide, + Nasal, + Fricative, + Closure, + Burst, + Silence, +} + +/// One synthesis target. +#[derive(Clone, Copy, Debug, PartialEq)] +struct Phone { + kind: Kind, + /// Formant frequencies (adult male, scaled per voice) and bandwidths, Hz. + f: [f32; 3], + bw: [f32; 3], + /// Voicing amplitude 0–1. + voice: f32, + /// Noise amplitude 0–1, through a resonator at `noise_f` / `noise_bw`. + noise: f32, + noise_f: f32, + noise_bw: f32, + dur_ms: f32, + /// Pitch multiplier at this phone (stress, sentence end). + pitch: f32, + /// Ends a sentence (pitch reset after it). + sentence_end: bool, +} + +const BW: [f32; 3] = [80.0, 100.0, 140.0]; +/// Neutral tract: what silences and closures move towards. +const NEUTRAL: [f32; 3] = [500.0, 1500.0, 2500.0]; + +fn base(kind: Kind, f: [f32; 3], dur_ms: f32) -> Phone { + Phone { kind, f, bw: BW, voice: 0.0, noise: 0.0, noise_f: 3000.0, noise_bw: 2000.0, dur_ms, pitch: 1.0, sentence_end: false } +} + +fn vowel(f: [f32; 3], dur_ms: f32) -> Phone { + Phone { voice: 1.0, ..base(Kind::Vowel, f, dur_ms) } +} + +fn glide(f: [f32; 3]) -> Phone { + Phone { voice: 0.8, ..base(Kind::Glide, f, 65.0) } +} + +fn nasal(f: [f32; 3]) -> Phone { + Phone { voice: 0.45, bw: [60.0, 200.0, 300.0], ..base(Kind::Nasal, f, 75.0) } +} + +fn fricative(noise_f: f32, noise_bw: f32, noise: f32, voiced: bool, dur_ms: f32) -> Phone { + Phone { voice: if voiced { 0.35 } else { 0.0 }, noise, noise_f, noise_bw, ..base(Kind::Fricative, NEUTRAL, dur_ms) } +} + +fn silence(dur_ms: f32) -> Phone { + base(Kind::Silence, NEUTRAL, dur_ms) +} + +/// A stop: closure then burst. +fn stop(out: &mut Vec, burst_f: f32, voiced: bool) { + out.push(Phone { voice: if voiced { 0.12 } else { 0.0 }, ..base(Kind::Closure, NEUTRAL, if voiced { 45.0 } else { 55.0 }) }); + out.push(Phone { noise: 0.9, noise_f: burst_f, noise_bw: 1800.0, ..base(Kind::Burst, NEUTRAL, if voiced { 15.0 } else { 30.0 }) }); +} + +// Peterson & Barney (1952) adult male averages: F1, F2, F3. +const AA: [f32; 3] = [730.0, 1090.0, 2440.0]; +const AE: [f32; 3] = [660.0, 1720.0, 2410.0]; +const AH: [f32; 3] = [640.0, 1190.0, 2390.0]; +const AO: [f32; 3] = [570.0, 840.0, 2410.0]; +const EH: [f32; 3] = [530.0, 1840.0, 2480.0]; +const ER: [f32; 3] = [490.0, 1350.0, 1690.0]; +const IH: [f32; 3] = [390.0, 1990.0, 2550.0]; +const IY: [f32; 3] = [270.0, 2290.0, 3010.0]; +const UH: [f32; 3] = [440.0, 1020.0, 2240.0]; +const UW: [f32; 3] = [300.0, 870.0, 2240.0]; + +fn diphthong(out: &mut Vec, a: [f32; 3], b: [f32; 3], stressed: bool) { + let d = if stressed { 95.0 } else { 70.0 }; + out.push(vowel(a, d)); + out.push(vowel(b, d * 0.8)); +} + +const DIGITS: [&str; 10] = ["zero", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"]; + +fn is_vowel(c: char) -> bool { + matches!(c, 'a' | 'e' | 'i' | 'o' | 'u') +} + +/// Phones for a piece of speech (no pause markers). +fn phones(text: &str) -> Vec { + let mut out = Vec::new(); + let mut word = String::new(); + let mut question_from = 0usize; + for c in text.chars().chain(std::iter::once(' ')) { + let c = c.to_ascii_lowercase(); + if c.is_ascii_alphabetic() || c == '\'' { + if c != '\'' { + word.push(c); + } + continue; + } + if !word.is_empty() { + speak_word(&word, &mut out); + word.clear(); + } + match c { + '0'..='9' => { + let d = DIGITS.get(c as usize - '0' as usize).copied().unwrap_or(""); + speak_word(d, &mut out); + out.push(silence(30.0)); + } + ',' | ';' | ':' | '-' | '(' | ')' => out.push(silence(170.0)), + '.' | '!' | '?' => { + if c == '?' { + // rise over the last word of the question + let n = out.len(); + let from = + out.get(question_from..).map_or(n, |s| s.iter().rposition(|p| p.kind == Kind::Silence).map_or(question_from, |i| question_from + i)); + for p in out.get_mut(from..).into_iter().flatten() { + p.pitch *= 1.25; + } + } + if let Some(last) = out.last_mut() { + last.sentence_end = true; + } + out.push(silence(330.0)); + question_from = out.len(); + } + _ => { + if out.last().is_some_and(|p| p.kind != Kind::Silence) { + out.push(silence(25.0)); + } + } + } + } + out +} + +/// Spelling rules for one lowercase ASCII word. +fn speak_word(w: &str, out: &mut Vec) { + let c: Vec = w.chars().collect(); + let n = c.len(); + let at = |i: usize| c.get(i).copied().unwrap_or(' '); + // a final silent e makes the vowel before the last consonant long: make, time, home, cute + let magic_e = n >= 3 && at(n - 1) == 'e' && !is_vowel(at(n - 2)) && is_vowel(at(n - 3)); + let mut stressed = true; + let mut i = 0; + while i < n { + let (a, b, d) = (at(i), at(i + 1), at(i + 2)); + let long = magic_e && i == n - 3; + let vd = if stressed { 115.0 } else { 80.0 }; + let before = out.len(); + let step = match a { + 'a' if b == 'i' || b == 'y' => { + diphthong(out, EH, IY, stressed); + 2 + } + 'a' if b == 'u' || b == 'w' => { + out.push(vowel(AO, vd)); + 2 + } + 'a' if b == 'r' => { + out.push(vowel(AA, vd)); + out.push(glide([420.0, 1300.0, 1600.0])); + 2 + } + 'a' if long => { + diphthong(out, EH, IY, stressed); + 1 + } + 'a' => { + out.push(vowel(if stressed { AE } else { AH }, vd)); + 1 + } + 'e' if b == 'e' || b == 'a' => { + out.push(vowel(IY, vd + 20.0)); + 2 + } + 'e' if b == 'r' => { + out.push(vowel(ER, vd)); + 2 + } + 'e' if b == 'y' && i + 2 == n => { + out.push(vowel(IY, vd)); + 2 + } + 'e' if i == n - 1 && n > 2 => 1, // silent final e + 'e' => { + out.push(vowel(if i == n - 1 { IY } else { EH }, vd)); + 1 + } + 'i' if b == 'g' && d == 'h' => { + diphthong(out, AA, IY, stressed); + 3 + } + 'i' if long || (i == n - 1 && n <= 2) => { + diphthong(out, AA, IY, stressed); + 1 + } + 'i' if b == 'r' => { + out.push(vowel(ER, vd)); + 2 + } + 'i' => { + out.push(vowel(IH, vd)); + 1 + } + 'o' if b == 'o' => { + out.push(vowel(if d == 'k' { UH } else { UW }, vd + 15.0)); + 2 + } + 'o' if b == 'u' || b == 'w' => { + diphthong(out, AA, UW, stressed); + 2 + } + 'o' if b == 'i' || b == 'y' => { + diphthong(out, AO, IY, stressed); + 2 + } + 'o' if b == 'a' || long || i == n - 1 => { + diphthong(out, AO, UW, stressed); + if b == 'a' { 2 } else { 1 } + } + 'o' if b == 'r' => { + out.push(vowel(AO, vd)); + out.push(glide([420.0, 1300.0, 1600.0])); + 2 + } + 'o' => { + out.push(vowel(AA, vd)); + 1 + } + 'u' if long => { + out.push(glide([280.0, 2200.0, 3000.0])); + out.push(vowel(UW, vd)); + 1 + } + 'u' if b == 'r' => { + out.push(vowel(ER, vd)); + 2 + } + 'u' => { + out.push(vowel(AH, vd)); + 1 + } + 'y' if i == 0 => { + out.push(glide([280.0, 2200.0, 3000.0])); + 1 + } + 'y' => { + if i == n - 1 && n <= 3 { + diphthong(out, AA, IY, stressed); + } else { + out.push(vowel(if i == n - 1 { IY } else { IH }, vd)); + } + 1 + } + 't' if b == 'h' => { + out.push(fricative(6000.0, 4000.0, 0.25, false, 80.0)); + 2 + } + 's' if b == 'h' => { + out.push(fricative(2600.0, 1400.0, 0.7, false, 110.0)); + 2 + } + 'c' if b == 'h' => { + stop(out, 4000.0, false); + out.push(fricative(2600.0, 1400.0, 0.7, false, 70.0)); + 2 + } + 'p' if b == 'h' => { + out.push(fricative(5000.0, 4000.0, 0.3, false, 90.0)); + 2 + } + 'w' if b == 'h' => { + out.push(glide([300.0, 610.0, 2200.0])); + 2 + } + 'n' if b == 'g' => { + out.push(nasal([250.0, 2300.0, 2700.0])); + 2 + } + 'c' if b == 'k' => { + stop(out, 2000.0, false); + 2 + } + 'q' if b == 'u' => { + stop(out, 2000.0, false); + out.push(glide([300.0, 610.0, 2200.0])); + 2 + } + 'c' if matches!(b, 'e' | 'i' | 'y') => { + out.push(fricative(6000.0, 2000.0, 0.6, false, 100.0)); + 1 + } + 'c' | 'k' | 'q' => { + stop(out, 2000.0, false); + 1 + } + 'g' if matches!(b, 'e' | 'i') && i > 0 => { + stop(out, 3500.0, true); + out.push(fricative(2600.0, 1400.0, 0.5, true, 60.0)); + 1 + } + 'g' => { + stop(out, 2000.0, true); + 1 + } + 'j' => { + stop(out, 3500.0, true); + out.push(fricative(2600.0, 1400.0, 0.5, true, 60.0)); + 1 + } + 'b' => { + stop(out, 800.0, true); + 1 + } + 'd' => { + stop(out, 3500.0, true); + 1 + } + 'p' => { + stop(out, 800.0, false); + 1 + } + 't' => { + stop(out, 4000.0, false); + 1 + } + 'x' => { + stop(out, 2000.0, false); + out.push(fricative(6000.0, 2000.0, 0.6, false, 90.0)); + 1 + } + 'f' => { + out.push(fricative(5000.0, 4000.0, 0.3, false, 90.0)); + 1 + } + 'v' => { + out.push(fricative(5000.0, 4000.0, 0.25, true, 70.0)); + 1 + } + 's' => { + out.push(fricative(6000.0, 2000.0, 0.6, false, 100.0)); + 1 + } + 'z' => { + out.push(fricative(6000.0, 2000.0, 0.45, true, 85.0)); + 1 + } + 'h' => { + out.push(fricative(1500.0, 2500.0, 0.3, false, 55.0)); + 1 + } + 'l' => { + out.push(glide([360.0, 1300.0, 2700.0])); + 1 + } + 'r' => { + out.push(glide([420.0, 1300.0, 1600.0])); + 1 + } + 'w' => { + out.push(glide([300.0, 610.0, 2200.0])); + 1 + } + 'm' => { + out.push(nasal([250.0, 1100.0, 2400.0])); + 1 + } + 'n' => { + out.push(nasal([250.0, 1700.0, 2600.0])); + 1 + } + _ => 1, + }; + let added_vowel = out.get(before..).is_some_and(|s| s.iter().any(|p| p.kind == Kind::Vowel)); + if added_vowel && stressed { + for p in out.get_mut(before..).into_iter().flatten() { + p.pitch *= 1.08; + } + stressed = false; + } + // doubled consonants are one sound: "ll", "ss", "tt" + let mut skip = step; + while !is_vowel(a) && at(i + skip) == a { + skip += 1; + } + i += skip; + } +} + +// ------------------------------------------------------------------------------------- render + +/// Two-pole resonator (unity gain at DC). +#[derive(Clone, Copy, Default)] +struct Resonator { + a: f32, + b: f32, + c: f32, + y1: f32, + y2: f32, +} + +impl Resonator { + fn set(&mut self, f: f32, bw: f32) { + let t = 1.0 / SAMPLE_RATE as f32; + let f = f.clamp(50.0, SAMPLE_RATE as f32 * 0.45); + let bw = bw.clamp(20.0, 8000.0); + self.c = -(-2.0 * std::f32::consts::PI * bw * t).exp(); + self.b = 2.0 * (-std::f32::consts::PI * bw * t).exp() * (2.0 * std::f32::consts::PI * f * t).cos(); + self.a = 1.0 - self.b - self.c; + } + fn tick(&mut self, x: f32) -> f32 { + let y = self.a * x + self.b * self.y1 + self.c * self.y2; + self.y2 = self.y1; + self.y1 = y; + y + } +} + +/// Deterministic white noise (xorshift32), −1..1. +struct Noise(u32); + +impl Noise { + fn next(&mut self) -> f32 { + let mut x = self.0; + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + self.0 = x; + (x as f32 / u32::MAX as f32) * 2.0 - 1.0 + } +} + +/// Parameters are updated every `BLOCK` samples. +const BLOCK: usize = 48; +/// Transition into each phone, ms. +const TRANSITION_MS: f32 = 30.0; + +fn lerp(a: f32, b: f32, t: f32) -> f32 { + a + (b - a) * t +} + +fn render(spec: &VoiceSpec, ph: &[Phone], params: &Params, out: &mut Vec) { + let sr = SAMPLE_RATE as f32; + let shift = 2f64.powf(params.semitones / 12.0); + let f0_base = (spec.f0 * shift) as f32; + let fscale = (spec.formant_scale * 2f64.powf(params.semitones / 12.0 * 0.15)) as f32; + let pace = params.pace as f32; + let mut res = [Resonator::default(); 3]; + let mut nres = Resonator::default(); + let mut noise = Noise(0x9E37_79B9); + let mut phase = 0.0f32; + let mut prev_glottal = 0.0f32; + let mut dc = (0.0f32, 0.0f32); + // sentence pitch line: from 1.1 down to 0.85 of the base over each sentence + let sentence_len = |from: usize| -> f32 { ph.get(from..).map_or(0, |s| s.iter().position(|p| p.sentence_end).map_or(s.len(), |e| e + 1)) as f32 }; + let mut sentence_start = 0usize; + let mut sentence_n = sentence_len(0).max(1.0); + let mut prev = silence(0.0); + for (k, p) in ph.iter().enumerate() { + let n = ((p.dur_ms / pace) / 1000.0 * sr).round().max(1.0) as usize; + let trans = ((TRANSITION_MS / pace) / 1000.0 * sr).min(n as f32 * 0.5).max(1.0); + let line = 1.1 - 0.25 * ((k - sentence_start) as f32 / sentence_n); + let mut i = 0; + while i < n { + let t = ((i as f32) / trans).min(1.0); + let f = [0, 1, 2].map(|j| lerp(prev.f[j], p.f[j], t) * fscale); + for (j, r) in res.iter_mut().enumerate() { + r.set(f[j], p.bw[j] * fscale.sqrt()); + } + nres.set(lerp(prev.noise_f, p.noise_f, t), lerp(prev.noise_bw, p.noise_bw, t)); + let voice = lerp(prev.voice, p.voice, t); + let namp = lerp(prev.noise, p.noise, t); + let f0 = (f0_base * line * lerp(prev.pitch, p.pitch, t)).clamp(40.0, 600.0); + let end = (i + BLOCK).min(n); + for _ in i..end { + phase += f0 / sr; + if phase >= 1.0 { + phase -= 1.0; + } + // glottal flow pulse (open 60 % of the period), differentiated for lip radiation + let g = if phase < 0.6 { (std::f32::consts::PI * phase / 0.6).sin().powi(2) } else { 0.0 }; + let dg = g - prev_glottal; + prev_glottal = g; + let breath = noise.next() * spec.breath * g; + let mut v = (dg * 6.0 + breath) * voice; + for r in &mut res { + v = r.tick(v); + } + let nz = nres.tick(noise.next()) * namp * 0.6; + // DC blocker + let x = v + nz; + let y = x - dc.0 + 0.995 * dc.1; + dc = (x, y); + out.push(y); + } + i = end; + } + prev = *p; + if p.sentence_end { + sentence_start = k + 1; + sentence_n = sentence_len(k + 1).max(1.0); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::voice; + + fn say(id: &str, text: &str, p: Params) -> Audio { + voice(id).unwrap().synthesize(text, &p).unwrap() + } + + /// Dominant pitch of the middle of `s` by autocorrelation, 60–500 Hz. + fn pitch(s: &[f32]) -> f64 { + let mid = &s[s.len() / 3..s.len() * 2 / 3]; + let sr = f64::from(SAMPLE_RATE); + let (lo, hi) = ((sr / 500.0) as usize, (sr / 60.0) as usize); + let best = (lo..hi).max_by(|&a, &b| { + let ca: f64 = mid.iter().zip(&mid[a..]).map(|(x, y)| f64::from(x * y)).sum(); + let cb: f64 = mid.iter().zip(&mid[b..]).map(|(x, y)| f64::from(x * y)).sum(); + ca.total_cmp(&cb) + }); + sr / best.unwrap() as f64 + } + + #[test] + fn speaks_deterministically_with_valid_samples() { + let a = say("basic-female", "Hello world, this is FilmCraft. Ready?", Params::default()); + let b = say("basic-female", "Hello world, this is FilmCraft. Ready?", Params::default()); + assert_eq!(a, b); + assert_eq!(a.sample_rate, SAMPLE_RATE); + assert!(a.seconds() > 1.0 && a.seconds() < 8.0, "{}", a.seconds()); + assert!(a.samples.iter().all(|s| s.is_finite() && s.abs() <= 0.8001)); + let rms = (a.samples.iter().map(|s| f64::from(s * s)).sum::() / a.samples.len() as f64).sqrt(); + assert!(rms > 0.03, "too quiet: {rms}"); + } + + #[test] + fn pause_markers_are_exact_silence() { + let plain = say("basic-male", "One. Two.", Params::default()); + let paused = say("basic-male", "One. [pause 1s] Two.", Params::default()); + assert_eq!(paused.samples.len(), plain.samples.len() + SAMPLE_RATE as usize); + let one = say("basic-male", "One. ", Params::default()).samples.len(); + assert!(paused.samples[one..one + SAMPLE_RATE as usize].iter().all(|&s| s == 0.0)); + } + + #[test] + fn pace_scales_duration() { + let text = "The quick brown fox jumps over the lazy dog."; + let n1 = say("basic-female", text, Params::default()).samples.len() as f64; + let n2 = say("basic-female", text, Params { pace: 2.0, ..Default::default() }).samples.len() as f64; + let nh = say("basic-female", text, Params { pace: 0.5, ..Default::default() }).samples.len() as f64; + assert!((n2 / n1 - 0.5).abs() < 0.05, "{}", n2 / n1); + assert!((nh / n1 - 2.0).abs() < 0.1, "{}", nh / n1); + } + + #[test] + fn pitch_follows_voice_and_setting() { + let text = "aaaaaaaaaaaaaaaaaaaa"; + let female = pitch(&say("basic-female", text, Params::default()).samples); + let male = pitch(&say("basic-male", text, Params::default()).samples); + let high = pitch(&say("basic-male", text, Params { semitones: 6.0, ..Default::default() }).samples); + assert!(female > male * 1.5, "female {female} male {male}"); + assert!((high / male - 2f64.powf(0.5)).abs() < 0.12, "high {high} male {male}"); + } + + #[test] + fn refuses_what_it_cannot_say() { + let v = voice("basic-female").unwrap(); + assert_eq!(v.synthesize("", &Params::default()), Err(TtsError::Empty)); + assert_eq!(v.synthesize("👍 🎬", &Params::default()), Err(TtsError::Empty)); + assert_eq!(v.synthesize("日本語", &Params::default()), Err(TtsError::Empty)); + assert_eq!(v.synthesize("[pause 2s]", &Params::default()), Err(TtsError::Empty)); + assert!(matches!(v.synthesize("hi", &Params { pace: f64::NAN, ..Default::default() }), Err(TtsError::Invalid(_)))); + // 64 KB of letters would be over the 30-minute limit: refused before rendering + assert_eq!(v.synthesize(&"a ".repeat(crate::MAX_TEXT_BYTES / 2), &Params { pace: 0.5, ..Default::default() }), Err(TtsError::AudioTooLong)); + } + + #[test] + fn hostile_scripts_never_panic() { + let v = voice("basic-male").unwrap(); + for s in ["a\0b", "x\u{301}y", "'''", "9999999999999999999999", "[pause 1s]a[pause", "Ünïcödé façade", "?!?!?!", "a?b?c?", "zzzzzzzz", "e"] { + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| v.synthesize(s, &Params::default()))); + assert!(r.is_ok(), "panicked on {s:?}"); + } + } +} diff --git a/crates/tts/src/kokoro/dsp.rs b/crates/tts/src/kokoro/dsp.rs new file mode 100644 index 000000000..7804ff957 --- /dev/null +++ b/crates/tts/src/kokoro/dsp.rs @@ -0,0 +1,163 @@ +//! The small DSP pieces of the vocoder: a 20-point STFT / inverse STFT (hop 5, periodic Hann +//! window, centred with reflection padding) and a seeded random source, so synthesis is +//! deterministic. + +/// FFT size, hop and the number of one-sided bins. +pub(crate) const N_FFT: usize = 20; +pub(crate) const HOP: usize = 5; +pub(crate) const BINS: usize = N_FFT / 2 + 1; + +fn hann() -> [f64; N_FFT] { + let mut w = [0f64; N_FFT]; + for (i, v) in w.iter_mut().enumerate() { + *v = 0.5 - 0.5 * (std::f64::consts::TAU * i as f64 / N_FFT as f64).cos(); + } + w +} + +/// Reflect index `i` (may be negative or past the end) into `0..n`. +fn reflect(i: isize, n: usize) -> usize { + if n <= 1 { + return 0; + } + let period = 2 * (n as isize - 1); + let mut j = i.rem_euclid(period); + if j >= n as isize { + j = period - j; + } + j as usize +} + +/// Magnitude and phase, bin-major: `mag[b * frames + f]`. Frames = `len / HOP + 1`. +pub(crate) fn stft(x: &[f32]) -> (Vec, Vec) { + let n = x.len(); + let frames = n / HOP + 1; + let w = hann(); + let half = (N_FFT / 2) as isize; + let mut mag = vec![0f32; BINS * frames]; + let mut ph = vec![0f32; BINS * frames]; + let mut seg = [0f64; N_FFT]; + for f in 0..frames { + for (k, s) in seg.iter_mut().enumerate() { + let i = (f * HOP) as isize + k as isize - half; + *s = f64::from(x.get(reflect(i, n)).copied().unwrap_or(0.0)) * w[k]; + } + for b in 0..BINS { + let (mut re, mut im) = (0f64, 0f64); + for (k, s) in seg.iter().enumerate() { + let a = -std::f64::consts::TAU * (b * k) as f64 / N_FFT as f64; + re += s * a.cos(); + im += s * a.sin(); + } + if let Some(m) = mag.get_mut(b * frames + f) { + *m = re.hypot(im) as f32; + } + if let Some(p) = ph.get_mut(b * frames + f) { + *p = im.atan2(re) as f32; + } + } + } + (mag, ph) +} + +/// Inverse of [`stft`] from magnitude and phase (bin-major); output length `HOP · (frames − 1)`. +pub(crate) fn istft(mag: &[f32], phase: &[f32]) -> Vec { + let frames = mag.len() / BINS; + if frames == 0 || phase.len() < mag.len() { + return Vec::new(); + } + let w = hann(); + let total = N_FFT + HOP * (frames - 1); + let mut out = vec![0f64; total]; + let mut env = vec![0f64; total]; + let mut frame = [0f64; N_FFT]; + for f in 0..frames { + // inverse real DFT of the one-sided spectrum + for (k, v) in frame.iter_mut().enumerate() { + let mut acc = 0f64; + for b in 0..BINS { + let m = f64::from(mag.get(b * frames + f).copied().unwrap_or(0.0)); + let p = f64::from(phase.get(b * frames + f).copied().unwrap_or(0.0)); + let (re, im) = (m * p.cos(), m * p.sin()); + let a = std::f64::consts::TAU * (b * k) as f64 / N_FFT as f64; + let c = if b == 0 || b == N_FFT / 2 { 1.0 } else { 2.0 }; + acc += c * (re * a.cos() - im * a.sin()); + } + *v = acc / N_FFT as f64; + } + for k in 0..N_FFT { + let at = f * HOP + k; + if let (Some(o), Some(e)) = (out.get_mut(at), env.get_mut(at)) { + *o += frame[k] * w[k]; + *e += w[k] * w[k]; + } + } + } + let start = N_FFT / 2; + let len = HOP * (frames - 1); + (start..start + len) + .map(|i| match (out.get(i), env.get(i)) { + (Some(o), Some(e)) if *e > 1e-11 => (o / e) as f32, + (Some(o), _) => *o as f32, + _ => 0.0, + }) + .collect() +} + +/// Deterministic random numbers (SplitMix64). +pub(crate) struct Rng(u64); + +impl Rng { + pub(crate) fn new(seed: u64) -> Rng { + Rng(seed ^ 0x9E37_79B9_7F4A_7C15) + } + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + /// Uniform in [0, 1). + pub(crate) fn uniform(&mut self) -> f64 { + (self.next() >> 11) as f64 / (1u64 << 53) as f64 + } + /// Standard normal (Box–Muller). + pub(crate) fn normal(&mut self) -> f64 { + let u1 = self.uniform().max(1e-12); + let u2 = self.uniform(); + (-2.0 * u1.ln()).sqrt() * (std::f64::consts::TAU * u2).cos() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn istft_inverts_stft() { + let x: Vec = (0..600).map(|i| ((i as f32) * 0.07).sin() * 0.5 + ((i as f32) * 0.31).cos() * 0.2).collect(); + let (m, p) = stft(&x); + assert_eq!(m.len(), BINS * (600 / HOP + 1)); + let y = istft(&m, &p); + assert_eq!(y.len(), 600); + let err = x.iter().zip(&y).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max); + assert!(err < 1e-4, "max error {err}"); + } + + #[test] + fn rng_is_deterministic_and_normalish() { + let mut a = Rng::new(7); + let mut b = Rng::new(7); + let xs: Vec = (0..20000).map(|_| a.normal()).collect(); + assert!((0..20000).all(|i| xs[i] == b.normal())); + let mean = xs.iter().sum::() / xs.len() as f64; + let var = xs.iter().map(|x| (x - mean).powi(2)).sum::() / xs.len() as f64; + assert!(mean.abs() < 0.03 && (var - 1.0).abs() < 0.05, "{mean} {var}"); + } + + #[test] + fn reflect_pads_like_numpy() { + assert_eq!((-3..8).map(|i| reflect(i, 5)).collect::>(), vec![3, 2, 1, 0, 1, 2, 3, 4, 3, 2, 1]); + } +} diff --git a/crates/tts/src/kokoro/layers.rs b/crates/tts/src/kokoro/layers.rs new file mode 100644 index 000000000..ec599fa6d --- /dev/null +++ b/crates/tts/src/kokoro/layers.rs @@ -0,0 +1,362 @@ +//! Building blocks shared by the Kokoro modules. Tensors are `[1, channels, time]` unless noted; +//! everything runs in f32 on the CPU. + +use candle_core::{D, Tensor}; + +use super::weights::{Weights, err}; +use crate::TtsError; + +pub(crate) type R = Result; + +/// 1-D convolution with optional bias. +pub(crate) struct Conv { + w: Tensor, + b: Option, + pad: usize, + stride: usize, + dil: usize, + groups: usize, +} + +impl Conv { + pub(crate) fn load(wt: &Weights, name: &str, pad: usize, stride: usize, dil: usize, groups: usize) -> R { + let w = wt.get(&format!("{name}.weight"))?; + if w.rank() != 3 { + return Err(err(format!("{name}: conv weight must be 3-D"))); + } + let b = wt.get(&format!("{name}.bias")).ok(); + Ok(Conv { w, b, pad, stride, dil, groups }) + } + /// `pad = dil · (k − 1) / 2` ("same" length for stride 1). + pub(crate) fn same(wt: &Weights, name: &str, dil: usize) -> R { + let k = wt.get(&format!("{name}.weight"))?.dims().get(2).copied().unwrap_or(1); + Conv::load(wt, name, dil * (k.saturating_sub(1)) / 2, 1, dil, 1) + } + pub(crate) fn fwd(&self, x: &Tensor) -> R { + let y = x.conv1d(&self.w, self.pad, self.stride, self.dil, self.groups).map_err(err)?; + add_bias(y, self.b.as_ref()) + } +} + +fn add_bias(y: Tensor, b: Option<&Tensor>) -> R { + match b { + Some(b) => { + let c = b.dims1().map_err(err)?; + y.broadcast_add(&b.reshape((1, c, 1)).map_err(err)?).map_err(err) + } + None => Ok(y), + } +} + +/// Transposed 1-D convolution. +pub(crate) struct ConvT { + w: Tensor, + b: Option, + pad: usize, + out_pad: usize, + stride: usize, + groups: usize, +} + +impl ConvT { + pub(crate) fn load(wt: &Weights, name: &str, pad: usize, out_pad: usize, stride: usize, groups: usize) -> R { + Ok(ConvT { w: wt.get(&format!("{name}.weight"))?, b: wt.get(&format!("{name}.bias")).ok(), pad, out_pad, stride, groups }) + } + pub(crate) fn fwd(&self, x: &Tensor) -> R { + let y = x.conv_transpose1d(&self.w, self.pad, self.out_pad, self.stride, 1, self.groups).map_err(err)?; + add_bias(y, self.b.as_ref()) + } +} + +/// Linear layer on the last dimension. +pub(crate) struct Linear { + w: Tensor, + b: Option, +} + +impl Linear { + pub(crate) fn load(wt: &Weights, name: &str) -> R { + Ok(Linear { w: wt.get(&format!("{name}.weight"))?.t().map_err(err)?, b: wt.get(&format!("{name}.bias")).ok() }) + } + pub(crate) fn fwd(&self, x: &Tensor) -> R { + let y = x.broadcast_matmul(&self.w).map_err(err)?; + match &self.b { + Some(b) => y.broadcast_add(b).map_err(err), + None => Ok(y), + } + } +} + +pub(crate) fn leaky(x: &Tensor, slope: f64) -> R { + // max(x, slope · x) for 0 < slope < 1 + x.maximum(&(x * slope).map_err(err)?).map_err(err) +} + +/// Normalise over the last dimension (layer norm without affine). +pub(crate) fn norm_last(x: &Tensor, eps: f64) -> R { + let mean = x.mean_keepdim(D::Minus1).map_err(err)?; + let xc = x.broadcast_sub(&mean).map_err(err)?; + let var = xc.sqr().map_err(err)?.mean_keepdim(D::Minus1).map_err(err)?; + xc.broadcast_div(&(var + eps).map_err(err)?.sqrt().map_err(err)?).map_err(err) +} + +/// Layer norm over the last dimension with weight and bias. +pub(crate) struct LayerNorm { + g: Tensor, + b: Tensor, + eps: f64, +} + +impl LayerNorm { + pub(crate) fn load(wt: &Weights, gamma: &str, beta: &str, eps: f64) -> R { + Ok(LayerNorm { g: wt.get(gamma)?, b: wt.get(beta)?, eps }) + } + pub(crate) fn fwd(&self, x: &Tensor) -> R { + norm_last(x, self.eps)?.broadcast_mul(&self.g).map_err(err)?.broadcast_add(&self.b).map_err(err) + } +} + +/// Split a style projection `[1, 2C]` into `(1 + gamma, beta)`. +fn style_affine(fc: &Linear, s: &Tensor) -> R<(Tensor, Tensor)> { + let h = fc.fwd(s)?; + let c2 = h.dim(1).map_err(err)?; + let c = c2 / 2; + let gamma = (h.narrow(1, 0, c).map_err(err)? + 1.0).map_err(err)?; + let beta = h.narrow(1, c, c).map_err(err)?; + Ok((gamma, beta)) +} + +/// Adaptive instance norm: instance-normalise each channel over time, then scale and shift by the +/// style (StyleTTS 2). +pub(crate) struct AdaIn { + fc: Linear, +} + +impl AdaIn { + pub(crate) fn load(wt: &Weights, name: &str) -> R { + Ok(AdaIn { fc: Linear::load(wt, &format!("{name}.fc"))? }) + } + pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R { + let (gamma, beta) = style_affine(&self.fc, s)?; + let c = gamma.dim(1).map_err(err)?; + let n = norm_last(x, 1e-5)?; + n.broadcast_mul(&gamma.reshape((1, c, 1)).map_err(err)?).map_err(err)?.broadcast_add(&beta.reshape((1, c, 1)).map_err(err)?).map_err(err) + } +} + +/// Adaptive layer norm on `[1, T, C]`: layer-normalise over channels, then style scale and shift. +pub(crate) struct AdaLayerNorm { + fc: Linear, +} + +impl AdaLayerNorm { + pub(crate) fn load(wt: &Weights, name: &str) -> R { + Ok(AdaLayerNorm { fc: Linear::load(wt, &format!("{name}.fc"))? }) + } + pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R { + let (gamma, beta) = style_affine(&self.fc, s)?; + norm_last(x, 1e-5)?.broadcast_mul(&gamma).map_err(err)?.broadcast_add(&beta).map_err(err) + } +} + +/// Residual block with adaptive instance norm (StyleTTS 2 `AdainResBlk1d`), optionally doubling +/// the time resolution. +pub(crate) struct AdaResBlock { + norm1: AdaIn, + norm2: AdaIn, + conv1: Conv, + conv2: Conv, + shortcut: Option, + pool: Option, +} + +impl AdaResBlock { + pub(crate) fn load(wt: &Weights, name: &str, upsample: bool) -> R { + let shortcut = wt.get(&format!("{name}.conv1x1.weight")).ok().map(|w| Conv { w, b: None, pad: 0, stride: 1, dil: 1, groups: 1 }); + let pool = if upsample { + let c = wt.get(&format!("{name}.pool.weight"))?.dim(0).map_err(err)?; + Some(ConvT::load(wt, &format!("{name}.pool"), 1, 1, 2, c)?) + } else { + None + }; + Ok(AdaResBlock { + norm1: AdaIn::load(wt, &format!("{name}.norm1"))?, + norm2: AdaIn::load(wt, &format!("{name}.norm2"))?, + conv1: Conv::same(wt, &format!("{name}.conv1"), 1)?, + conv2: Conv::same(wt, &format!("{name}.conv2"), 1)?, + shortcut, + pool, + }) + } + pub(crate) fn upsamples(&self) -> bool { + self.pool.is_some() + } + pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R { + let mut r = leaky(&self.norm1.fwd(x, s)?, 0.2)?; + if let Some(p) = &self.pool { + r = p.fwd(&r)?; + } + r = self.conv1.fwd(&r)?; + r = leaky(&self.norm2.fwd(&r, s)?, 0.2)?; + r = self.conv2.fwd(&r)?; + let mut sc = x.clone(); + if self.pool.is_some() { + sc = upsample2(&sc)?; + } + if let Some(c) = &self.shortcut { + sc = c.fwd(&sc)?; + } + ((r + sc).map_err(err)? / std::f64::consts::SQRT_2).map_err(err) + } +} + +/// Nearest-neighbour ×2 along time. +pub(crate) fn upsample2(x: &Tensor) -> R { + let (b, c, t) = x.dims3().map_err(err)?; + x.unsqueeze(3).map_err(err)?.broadcast_as((b, c, t, 2)).map_err(err)?.reshape((b, c, t * 2)).map_err(err) +} + +/// Snake activation `x + sin²(αx) / α` with a learned α per channel. +pub(crate) fn snake(x: &Tensor, alpha: &Tensor) -> R { + let ax = x.broadcast_mul(alpha).map_err(err)?; + let s2 = ax.sin().map_err(err)?.sqr().map_err(err)?; + let inv = (alpha.clone() + 1e-9).map_err(err)?.recip().map_err(err)?; + (x + s2.broadcast_mul(&inv).map_err(err)?).map_err(err) +} + +/// Generator residual block (HiFi-GAN "resblock 1" with adaptive instance norm and Snake). +pub(crate) struct SnakeResBlock { + convs1: Vec, + convs2: Vec, + adain1: Vec, + adain2: Vec, + alpha1: Vec, + alpha2: Vec, +} + +impl SnakeResBlock { + pub(crate) fn load(wt: &Weights, name: &str, dilations: [usize; 3]) -> R { + let mut b = SnakeResBlock { convs1: vec![], convs2: vec![], adain1: vec![], adain2: vec![], alpha1: vec![], alpha2: vec![] }; + for (j, d) in dilations.iter().enumerate() { + b.convs1.push(Conv::same(wt, &format!("{name}.convs1.{j}"), *d)?); + b.convs2.push(Conv::same(wt, &format!("{name}.convs2.{j}"), 1)?); + b.adain1.push(AdaIn::load(wt, &format!("{name}.adain1.{j}"))?); + b.adain2.push(AdaIn::load(wt, &format!("{name}.adain2.{j}"))?); + b.alpha1.push(wt.get(&format!("{name}.alpha1.{j}"))?); + b.alpha2.push(wt.get(&format!("{name}.alpha2.{j}"))?); + } + Ok(b) + } + pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R { + let mut x = x.clone(); + for j in 0..self.convs1.len() { + let (Some(c1), Some(c2), Some(n1), Some(n2), Some(a1), Some(a2)) = + (self.convs1.get(j), self.convs2.get(j), self.adain1.get(j), self.adain2.get(j), self.alpha1.get(j), self.alpha2.get(j)) + else { + break; + }; + let mut t = snake(&n1.fwd(&x, s)?, a1)?; + t = c1.fwd(&t)?; + t = snake(&n2.fwd(&t, s)?, a2)?; + t = c2.fwd(&t)?; + x = (x + t).map_err(err)?; + } + Ok(x) + } +} + +/// One LSTM direction (PyTorch gate order i, f, g, o), run in plain loops for speed. +struct LstmDir { + w_ih: Tensor, + w_hh: Vec, + bias: Vec, + hidden: usize, +} + +impl LstmDir { + fn load(wt: &Weights, name: &str, suffix: &str) -> R { + let w_ih = wt.get(&format!("{name}.weight_ih_l0{suffix}"))?; + let w_hh = wt.get(&format!("{name}.weight_hh_l0{suffix}"))?; + let (g4, hidden) = w_hh.dims2().map_err(err)?; + if g4 != 4 * hidden { + return Err(err(format!("{name}: bad LSTM shape"))); + } + let b_ih: Vec = wt.get(&format!("{name}.bias_ih_l0{suffix}"))?.to_vec1().map_err(err)?; + let b_hh: Vec = wt.get(&format!("{name}.bias_hh_l0{suffix}"))?.to_vec1().map_err(err)?; + let bias = b_ih.iter().zip(&b_hh).map(|(a, b)| a + b).collect(); + // row-major [4H, H] → transposed [H, 4H] so the recurrent product walks memory forwards + let hh: Vec> = w_hh.to_vec2().map_err(err)?; + let mut w_t = vec![0f32; hidden * g4]; + for (r, row) in hh.iter().enumerate() { + for (c, v) in row.iter().enumerate() { + if let Some(slot) = w_t.get_mut(c * g4 + r) { + *slot = *v; + } + } + } + Ok(LstmDir { w_ih: w_ih.t().map_err(err)?, w_hh: w_t, bias, hidden }) + } + + /// `x`: `[T, I]` → `[T, H]` (in input order). + fn run(&self, x: &Tensor, reverse: bool) -> R> { + let (t_len, _) = x.dims2().map_err(err)?; + let h = self.hidden; + let g4 = 4 * h; + let pre: Vec = x.matmul(&self.w_ih).map_err(err)?.flatten_all().map_err(err)?.to_vec1().map_err(err)?; + let mut out = vec![0f32; t_len * h]; + let (mut hs, mut cs) = (vec![0f32; h], vec![0f32; h]); + let mut gates = vec![0f32; g4]; + let sig = |v: f32| 1.0 / (1.0 + (-v).exp()); + for step in 0..t_len { + let t = if reverse { t_len - 1 - step } else { step }; + let Some(p) = pre.get(t * g4..(t + 1) * g4) else { break }; + for ((g, a), b) in gates.iter_mut().zip(p).zip(&self.bias) { + *g = a + b; + } + for (k, hv) in hs.iter().enumerate() { + if *hv == 0.0 { + continue; + } + let Some(row) = self.w_hh.get(k * g4..(k + 1) * g4) else { break }; + for (g, w) in gates.iter_mut().zip(row) { + *g += hv * w; + } + } + for j in 0..h { + let (i, f, gg, o) = (gates[j], gates[h + j], gates[2 * h + j], gates[3 * h + j]); + let c = sig(f) * cs[j] + sig(i) * gg.tanh(); + cs[j] = c; + hs[j] = sig(o) * c.tanh(); + } + if let Some(slot) = out.get_mut(t * h..(t + 1) * h) { + slot.copy_from_slice(&hs); + } + } + Ok(out) + } +} + +/// Bidirectional single-layer LSTM on `[1, T, I]` → `[1, T, 2H]`. +pub(crate) struct BiLstm { + fwd: LstmDir, + bwd: LstmDir, +} + +impl BiLstm { + pub(crate) fn load(wt: &Weights, name: &str) -> R { + Ok(BiLstm { fwd: LstmDir::load(wt, name, "")?, bwd: LstmDir::load(wt, name, "_reverse")? }) + } + pub(crate) fn fwd(&self, x: &Tensor) -> R { + let x2 = x.squeeze(0).map_err(err)?.contiguous().map_err(err)?; + let t = x2.dim(0).map_err(err)?; + let h = self.fwd.hidden; + let a = self.fwd.run(&x2, false)?; + let b = self.bwd.run(&x2, true)?; + let mut out = Vec::with_capacity(t * 2 * h); + for i in 0..t { + out.extend_from_slice(a.get(i * h..(i + 1) * h).unwrap_or(&[])); + out.extend_from_slice(b.get(i * h..(i + 1) * h).unwrap_or(&[])); + } + Tensor::from_vec(out, (1, t, 2 * h), x.device()).map_err(err) + } +} diff --git a/crates/tts/src/kokoro/mod.rs b/crates/tts/src/kokoro/mod.rs new file mode 100644 index 000000000..dd392efd6 --- /dev/null +++ b/crates/tts/src/kokoro/mod.rs @@ -0,0 +1,113 @@ +//! Kokoro-82M neural voices (feature `kokoro`): pure-Rust inference on candle (CPU). +//! +//! Implemented from the papers the model card cites (StyleTTS 2, iSTFTNet; with ALBERT, HiFi-GAN, +//! the neural source-filter model and the Snake activation they build on), the released +//! `config.json` and the tensor names and shapes in the checkpoint. No reference code was read. +//! Weights (Apache-2.0, hexgrad) are downloaded on first use and never bundled. +//! +//! [`Kokoro::speak`] takes Kokoro phonemes (from `filmcraft-tts-text`), at most +//! [`MAX_PHONEMES`] per call, and a voice pack. + +mod dsp; +mod layers; +mod model; +mod weights; + +use std::path::Path; + +use crate::TtsError; + +pub use model::SAMPLE_RATE as KOKORO_RATE; + +/// Longest phoneme string one forward pass takes (the position table has 512 rows, two of them +/// for the boundary tokens). +pub const MAX_PHONEMES: usize = 510; + +/// A loaded Kokoro model. +pub struct Kokoro { + model: model::Model, + vocab: std::collections::HashMap, +} + +/// A loaded voice pack. +pub struct VoicePack(Vec); + +impl VoicePack { + pub fn load(path: &Path) -> Result { + weights::read_voice(path).map(VoicePack) + } +} + +impl Kokoro { + /// Load `config.json` and `kokoro-v1_0.pth` from `dir`. + pub fn load(dir: &Path) -> Result { + let cfg = weights::read_config(&dir.join("config.json"))?; + let w = weights::Weights::load(&dir.join("kokoro-v1_0.pth"))?; + Ok(Kokoro { model: model::Model::load(&w)?, vocab: cfg.vocab }) + } + + /// Phoneme ids of `phonemes`; symbols outside the vocabulary are dropped. + pub fn ids(&self, phonemes: &str) -> Vec { + phonemes.chars().filter_map(|c| self.vocab.get(&c).copied()).collect() + } + + /// Speak `phonemes` with `voice` at `speed` (0.5–2). Deterministic for a given `seed`. + pub fn speak(&self, phonemes: &str, voice: &VoicePack, speed: f64, seed: u64) -> Result, TtsError> { + if !(0.5..=2.0).contains(&speed) { + return Err(TtsError::Invalid("speed must be between 0.5 and 2".into())); + } + let ids = self.ids(phonemes); + if ids.is_empty() { + return Err(TtsError::Empty); + } + if ids.len() > MAX_PHONEMES { + return Err(TtsError::Invalid(format!("at most {MAX_PHONEMES} phonemes per pass"))); + } + let style = weights::style_for(&voice.0, ids.len(), self.model.device())?; + let out = self.model.forward(&ids, &style, speed, seed)?; + if out.iter().any(|s| !s.is_finite()) { + return Err(TtsError::Model("the model produced invalid samples".into())); + } + Ok(out) + } +} + +/// Tests with the real model run when `FILMCRAFT_KOKORO_DIR` names a directory holding +/// `config.json`, `kokoro-v1_0.pth` and `af_heart.pt` (the downloaded package); they skip otherwise. +#[cfg(test)] +mod tests { + use super::*; + + fn model() -> Option<(Kokoro, VoicePack)> { + let dir = std::path::PathBuf::from(std::env::var_os("FILMCRAFT_KOKORO_DIR")?); + let voice = [dir.join("af_heart.pt"), dir.join("voices/af_heart.pt")].into_iter().find(|p| p.exists())?; + Some((Kokoro::load(&dir).unwrap(), VoicePack::load(&voice).unwrap())) + } + + #[test] + fn speaks_deterministically_and_pace_changes_the_length() { + let Some((k, v)) = model() else { return }; + let ph = "həlˈO wˈɜɹld."; + let a = k.speak(ph, &v, 1.0, 1).unwrap(); + assert_eq!(a, k.speak(ph, &v, 1.0, 1).unwrap(), "deterministic"); + assert_eq!(a.len() % 600, 0, "whole frames"); + let secs = a.len() as f64 / f64::from(KOKORO_RATE); + assert!((0.5..3.0).contains(&secs), "{secs}"); + let peak = a.iter().fold(0f32, |m, x| m.max(x.abs())); + assert!(peak > 0.05 && peak < 1.0, "{peak}"); + let fast = k.speak(ph, &v, 2.0, 1).unwrap(); + let ratio = fast.len() as f64 / a.len() as f64; + assert!((0.35..0.7).contains(&ratio), "{ratio}"); + } + + #[test] + fn refuses_bad_input() { + let Some((k, v)) = model() else { return }; + assert_eq!(k.speak("", &v, 1.0, 1).err(), Some(TtsError::Empty)); + assert_eq!(k.speak("日本語👍", &v, 1.0, 1).err(), Some(TtsError::Empty), "nothing in the vocabulary"); + assert!(k.speak("a", &v, 3.0, 1).is_err()); + assert!(k.speak("a", &v, f64::NAN, 1).is_err()); + assert!(k.speak(&"ə".repeat(MAX_PHONEMES + 1), &v, 1.0, 1).is_err()); + assert_eq!(k.ids("a日b"), k.ids("ab")); + } +} diff --git a/crates/tts/src/kokoro/model.rs b/crates/tts/src/kokoro/model.rs new file mode 100644 index 000000000..777c8f771 --- /dev/null +++ b/crates/tts/src/kokoro/model.rs @@ -0,0 +1,385 @@ +//! The Kokoro-82M forward pass (inference only, batch of one). +//! +//! 1. **Text** – phoneme ids go through an ALBERT encoder (one shared layer applied 12 times) and +//! a linear map to 512 channels (`bert`, `bert_encoder`), and separately through an embedding, +//! three convolutions and a bidirectional LSTM (`text_encoder`). +//! 2. **Prosody** – the duration encoder (LSTMs with style-adaptive layer norm), a duration head +//! (50 bins, sigmoid, summed), then pitch (F0) and energy (N) curves at twice the frame rate from +//! adaptive residual blocks (`predictor`). +//! 3. **Decoder** – text features expanded to frames by the durations, combined with F0 and N in +//! adaptive residual blocks (`decoder.encode`, `decoder.decode`). +//! 4. **Vocoder** – iSTFTNet: a harmonic-plus-noise source from F0 (9 harmonics), two upsampling +//! stages (×10, ×6) with source injection and Snake residual blocks, then magnitude and phase for +//! an inverse STFT (n_fft 20, hop 5) at 24 kHz. +//! +//! Style: a voice pack row of 256 values; the first 128 condition the decoder, the last 128 the +//! prosody predictor. + +use candle_core::{D, Device, Tensor}; + +use super::layers::*; +use super::weights::{Weights, err}; + +const HEADS: usize = 12; +const LAYERS: usize = 12; +pub const SAMPLE_RATE: u32 = 24_000; +/// Output samples per predicted duration frame (2 F0 frames × 60 upsampling × hop 5). +pub(crate) const SAMPLES_PER_FRAME: usize = 600; + +struct Albert { + word: Tensor, + pos: Tensor, + ttype: Tensor, + emb_ln: LayerNorm, + map_in: Linear, + q: Linear, + k: Linear, + v: Linear, + dense: Linear, + attn_ln: LayerNorm, + ffn: Linear, + ffn_out: Linear, + full_ln: LayerNorm, +} + +impl Albert { + fn load(w: &Weights) -> R { + let l = "bert.encoder.albert_layer_groups.0.albert_layers.0"; + Ok(Albert { + word: w.shaped("bert.embeddings.word_embeddings.weight", &[178, 128])?, + pos: w.shaped("bert.embeddings.position_embeddings.weight", &[512, 128])?, + ttype: w.shaped("bert.embeddings.token_type_embeddings.weight", &[2, 128])?, + emb_ln: LayerNorm::load(w, "bert.embeddings.LayerNorm.weight", "bert.embeddings.LayerNorm.bias", 1e-12)?, + map_in: Linear::load(w, "bert.encoder.embedding_hidden_mapping_in")?, + q: Linear::load(w, &format!("{l}.attention.query"))?, + k: Linear::load(w, &format!("{l}.attention.key"))?, + v: Linear::load(w, &format!("{l}.attention.value"))?, + dense: Linear::load(w, &format!("{l}.attention.dense"))?, + attn_ln: LayerNorm::load(w, &format!("{l}.attention.LayerNorm.weight"), &format!("{l}.attention.LayerNorm.bias"), 1e-12)?, + ffn: Linear::load(w, &format!("{l}.ffn"))?, + ffn_out: Linear::load(w, &format!("{l}.ffn_output"))?, + full_ln: LayerNorm::load(w, &format!("{l}.full_layer_layer_norm.weight"), &format!("{l}.full_layer_layer_norm.bias"), 1e-12)?, + }) + } + + /// `ids`: `[T]` → `[1, T, 768]`. + fn fwd(&self, ids: &Tensor) -> R { + let t = ids.dim(0).map_err(err)?; + let e = self.word.index_select(ids, 0).map_err(err)?; + let e = (e + self.pos.narrow(0, 0, t).map_err(err)?).map_err(err)?; + let e = e.broadcast_add(&self.ttype.narrow(0, 0, 1).map_err(err)?).map_err(err)?; + let mut x = self.map_in.fwd(&self.emb_ln.fwd(&e)?)?.unsqueeze(0).map_err(err)?; // [1, T, 768] + let hd = 768 / HEADS; + let scale = 1.0 / (hd as f64).sqrt(); + let split = |y: Tensor| -> R { y.reshape((1, t, HEADS, hd)).map_err(err)?.transpose(1, 2).map_err(err)?.contiguous().map_err(err) }; + for _ in 0..LAYERS { + let q = split(self.q.fwd(&x)?)?; + let k = split(self.k.fwd(&x)?)?; + let v = split(self.v.fwd(&x)?)?; + let att = (q.matmul(&k.t().map_err(err)?).map_err(err)? * scale).map_err(err)?; + let att = candle_nn::ops::softmax_last_dim(&att).map_err(err)?; + let ctx = att.matmul(&v).map_err(err)?.transpose(1, 2).map_err(err)?.reshape((1, t, 768)).map_err(err)?; + let a = self.attn_ln.fwd(&(x + self.dense.fwd(&ctx)?).map_err(err)?)?; + let f = self.ffn_out.fwd(&self.ffn.fwd(&a)?.gelu().map_err(err)?)?; + x = self.full_ln.fwd(&(f + a).map_err(err)?)?; + } + Ok(x) + } +} + +struct TextEncoder { + emb: Tensor, + convs: Vec<(Conv, LayerNorm)>, + lstm: BiLstm, +} + +impl TextEncoder { + fn load(w: &Weights) -> R { + let mut convs = Vec::new(); + for i in 0..3 { + let c = Conv::same(w, &format!("text_encoder.cnn.{i}.0"), 1)?; + let n = LayerNorm::load(w, &format!("text_encoder.cnn.{i}.1.gamma"), &format!("text_encoder.cnn.{i}.1.beta"), 1e-5)?; + convs.push((c, n)); + } + Ok(TextEncoder { emb: w.shaped("text_encoder.embedding.weight", &[178, 512])?, convs, lstm: BiLstm::load(w, "text_encoder.lstm")? }) + } + /// `[T]` → `[1, 512, T]`. + fn fwd(&self, ids: &Tensor) -> R { + let mut x = self.emb.index_select(ids, 0).map_err(err)?.t().map_err(err)?.unsqueeze(0).map_err(err)?; // [1, 512, T] + for (c, n) in &self.convs { + x = c.fwd(&x)?; + // layer norm over channels + x = n.fwd(&x.transpose(1, 2).map_err(err)?)?.transpose(1, 2).map_err(err)?; + x = leaky(&x, 0.2)?; + } + let y = self.lstm.fwd(&x.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?)?; // [1, T, 512] + y.transpose(1, 2).map_err(err) + } +} + +struct Predictor { + dur_lstms: Vec, + dur_norms: Vec, + lstm: BiLstm, + dur_proj: Linear, + shared: BiLstm, + f0: Vec, + n: Vec, + f0_proj: Conv, + n_proj: Conv, +} + +impl Predictor { + fn load(w: &Weights) -> R { + let mut dur_lstms = Vec::new(); + let mut dur_norms = Vec::new(); + for i in 0..3 { + dur_lstms.push(BiLstm::load(w, &format!("predictor.text_encoder.lstms.{}", 2 * i))?); + dur_norms.push(AdaLayerNorm::load(w, &format!("predictor.text_encoder.lstms.{}", 2 * i + 1))?); + } + let blocks = |k: &str| -> R> { (0..3).map(|i| AdaResBlock::load(w, &format!("predictor.{k}.{i}"), i == 1)).collect() }; + Ok(Predictor { + dur_lstms, + dur_norms, + lstm: BiLstm::load(w, "predictor.lstm")?, + dur_proj: Linear::load(w, "predictor.duration_proj.linear_layer")?, + shared: BiLstm::load(w, "predictor.shared")?, + f0: blocks("F0")?, + n: blocks("N")?, + f0_proj: Conv::load(w, "predictor.F0_proj", 0, 1, 1, 1)?, + n_proj: Conv::load(w, "predictor.N_proj", 0, 1, 1, 1)?, + }) + } + + /// Duration encoder: `d_en` `[1, 512, T]`, style `[1, 128]` → `[1, T, 640]`. + fn encode(&self, d_en: &Tensor, s: &Tensor) -> R { + let t = d_en.dim(2).map_err(err)?; + let s_t = s.unsqueeze(1).map_err(err)?.broadcast_as((1, t, 128)).map_err(err)?.contiguous().map_err(err)?; + let mut x = Tensor::cat(&[&d_en.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?, &s_t], 2).map_err(err)?; + for (l, n) in self.dur_lstms.iter().zip(&self.dur_norms) { + let y = n.fwd(&l.fwd(&x)?, s)?; + x = Tensor::cat(&[&y, &s_t], 2).map_err(err)?; + } + Ok(x) + } + + /// Frames per input token, at `speed`. + fn durations(&self, d: &Tensor, speed: f64) -> R> { + let x = self.lstm.fwd(d)?; + let logits = self.dur_proj.fwd(&x)?; + let dur = candle_nn::ops::sigmoid(&logits).map_err(err)?.sum(D::Minus1).map_err(err)?.squeeze(0).map_err(err)?; + let v: Vec = dur.to_vec1().map_err(err)?; + Ok(v.iter().map(|d| ((f64::from(*d) / speed).round().clamp(1.0, 200.0)) as usize).collect()) + } + + /// F0 and N curves `[1, 1, 2F]` from frame features `en` `[1, 640, F]`. + fn f0_n(&self, en: &Tensor, s: &Tensor) -> R<(Tensor, Tensor)> { + let x = self.shared.fwd(&en.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?)?.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?; + let mut f = x.clone(); + for b in &self.f0 { + f = b.fwd(&f, s)?; + } + let mut n = x; + for b in &self.n { + n = b.fwd(&n, s)?; + } + Ok((self.f0_proj.fwd(&f)?, self.n_proj.fwd(&n)?)) + } +} + +struct Generator { + l_linear: Linear, + noise_convs: Vec, + noise_res: Vec, + ups: Vec, + resblocks: Vec, + conv_post: Conv, +} + +impl Generator { + fn load(w: &Weights) -> R { + let g = "decoder.generator"; + let dil = [1, 3, 5]; + Ok(Generator { + l_linear: Linear::load(w, &format!("{g}.m_source.l_linear"))?, + noise_convs: vec![Conv::load(w, &format!("{g}.noise_convs.0"), 3, 6, 1, 1)?, Conv::load(w, &format!("{g}.noise_convs.1"), 0, 1, 1, 1)?], + noise_res: vec![SnakeResBlock::load(w, &format!("{g}.noise_res.0"), dil)?, SnakeResBlock::load(w, &format!("{g}.noise_res.1"), dil)?], + ups: vec![ConvT::load(w, &format!("{g}.ups.0"), 5, 0, 10, 1)?, ConvT::load(w, &format!("{g}.ups.1"), 3, 0, 6, 1)?], + resblocks: (0..6).map(|i| SnakeResBlock::load(w, &format!("{g}.resblocks.{i}"), dil)).collect::>()?, + conv_post: Conv::same(w, &format!("{g}.conv_post"), 1)?, + }) + } + + /// `x` `[1, 512, 2F]`, style `[1, 128]`, F0 per half-frame → samples. + fn fwd(&self, x: &Tensor, s: &Tensor, f0: &[f32], seed: u64) -> R> { + let dev = x.device(); + let source = harmonic_source(f0, &self.l_linear, seed, dev)?; + let (mag, phase) = super::dsp::stft(&source); + let frames = mag.len() / super::dsp::BINS; + let mut har = mag; + har.extend_from_slice(&phase); + let har = Tensor::from_vec(har, (1, 2 * super::dsp::BINS, frames), dev).map_err(err)?; + let mut x = x.clone(); + for i in 0..2 { + let (Some(nc), Some(nr), Some(up)) = (self.noise_convs.get(i), self.noise_res.get(i), self.ups.get(i)) else { break }; + x = leaky(&x, 0.1)?; + let xs = nr.fwd(&nc.fwd(&har)?, s)?; + x = up.fwd(&x)?; + if i == 1 { + // reflection pad one frame on the left + let first = x.narrow(2, 1, 1).map_err(err)?; + x = Tensor::cat(&[&first, &x], 2).map_err(err)?; + } + let t = x.dim(2).map_err(err)?.min(xs.dim(2).map_err(err)?); + x = (x.narrow(2, 0, t).map_err(err)? + xs.narrow(2, 0, t).map_err(err)?).map_err(err)?; + let mut acc: Option = None; + for j in 0..3 { + let Some(rb) = self.resblocks.get(i * 3 + j) else { break }; + let y = rb.fwd(&x, s)?; + acc = Some(match acc { + Some(a) => (a + y).map_err(err)?, + None => y, + }); + } + x = (acc.ok_or_else(|| err("missing resblocks"))? / 3.0).map_err(err)?; + } + x = leaky(&x, 0.01)?; + let y = self.conv_post.fwd(&x)?.squeeze(0).map_err(err)?; // [22, T] + let bins = super::dsp::BINS; + let mag: Vec = y.narrow(0, 0, bins).map_err(err)?.exp().map_err(err)?.flatten_all().map_err(err)?.to_vec1().map_err(err)?; + let ph: Vec = y.narrow(0, bins, bins).map_err(err)?.sin().map_err(err)?.flatten_all().map_err(err)?.to_vec1().map_err(err)?; + Ok(super::dsp::istft(&mag, &ph)) + } +} + +/// Neural source filter: 9 sine harmonics of F0 (voiced) or noise (unvoiced), mixed to one +/// channel by a learned linear layer and tanh. F0 is given per half-frame and held for 300 samples. +fn harmonic_source(f0: &[f32], l_linear: &Linear, seed: u64, dev: &Device) -> R> { + const HARM: usize = 9; + const HOLD: usize = SAMPLES_PER_FRAME / 2; + let n = f0.len() * HOLD; + let mut rng = super::dsp::Rng::new(seed); + // random start phase for the overtones, none for the fundamental + let start: Vec = (0..HARM).map(|h| if h == 0 { 0.0 } else { rng.uniform() }).collect(); + let mut phase = start; + let mut sines = vec![0f32; n * HARM]; + for (i, row) in sines.as_chunks_mut::().0.iter_mut().enumerate() { + let f = f64::from(f0.get(i / HOLD).copied().unwrap_or(0.0)).max(0.0); + let voiced = f > 10.0; + let noise_amp = if voiced { 0.003 } else { 0.1 / 3.0 }; + for (h, out) in row.iter_mut().enumerate() { + phase[h] = (phase[h] + f * (h + 1) as f64 / f64::from(SAMPLE_RATE)).fract(); + let sine = if voiced { 0.1 * (std::f64::consts::TAU * phase[h]).sin() } else { 0.0 }; + *out = (sine + noise_amp * rng.normal()) as f32; + } + } + let t = Tensor::from_vec(sines, (n, HARM), dev).map_err(err)?; + let y = l_linear.fwd(&t)?.tanh().map_err(err)?.flatten_all().map_err(err)?; + y.to_vec1().map_err(err) +} + +struct Decoder { + f0_conv: Conv, + n_conv: Conv, + encode: AdaResBlock, + decode: Vec, + asr_res: Conv, + vocoder: Generator, +} + +impl Decoder { + fn load(w: &Weights) -> R { + Ok(Decoder { + f0_conv: Conv::load(w, "decoder.F0_conv", 1, 2, 1, 1)?, + n_conv: Conv::load(w, "decoder.N_conv", 1, 2, 1, 1)?, + encode: AdaResBlock::load(w, "decoder.encode", false)?, + decode: (0..4).map(|i| AdaResBlock::load(w, &format!("decoder.decode.{i}"), i == 3)).collect::>()?, + asr_res: Conv::load(w, "decoder.asr_res.0", 0, 1, 1, 1)?, + vocoder: Generator::load(w)?, + }) + } + + fn fwd(&self, asr: &Tensor, f0: &Tensor, n: &Tensor, s: &Tensor, seed: u64) -> R> { + let f0_curve: Vec = f0.flatten_all().map_err(err)?.to_vec1().map_err(err)?; + let f = self.f0_conv.fwd(f0)?; + let nn = self.n_conv.fwd(n)?; + let frames = asr.dim(2).map_err(err)?.min(f.dim(2).map_err(err)?).min(nn.dim(2).map_err(err)?); + let asr = asr.narrow(2, 0, frames).map_err(err)?; + let f = f.narrow(2, 0, frames).map_err(err)?; + let nn = nn.narrow(2, 0, frames).map_err(err)?; + let mut x = self.encode.fwd(&Tensor::cat(&[&asr, &f, &nn], 1).map_err(err)?, s)?; + let res = self.asr_res.fwd(&asr)?; + let mut concat = true; + for b in &self.decode { + if concat { + x = Tensor::cat(&[&x, &res, &f, &nn], 1).map_err(err)?; + } + x = b.fwd(&x, s)?; + if b.upsamples() { + concat = false; + } + } + self.vocoder.fwd(&x, s, &f0_curve, seed) + } +} + +/// The loaded model. +pub(crate) struct Model { + albert: Albert, + bert_enc: Linear, + text: TextEncoder, + pred: Predictor, + dec: Decoder, + dev: Device, +} + +impl Model { + pub(crate) fn load(w: &Weights) -> R { + Ok(Model { + albert: Albert::load(w)?, + bert_enc: Linear::load(w, "bert_encoder")?, + text: TextEncoder::load(w)?, + pred: Predictor::load(w)?, + dec: Decoder::load(w)?, + dev: Device::Cpu, + }) + } + + pub(crate) fn device(&self) -> &Device { + &self.dev + } + + /// Phoneme ids (without the boundary tokens; at most 510) and a 256-value style → samples. + pub(crate) fn forward(&self, ids: &[u32], style: &Tensor, speed: f64, seed: u64) -> R> { + let mut padded = Vec::with_capacity(ids.len() + 2); + padded.push(0u32); + padded.extend_from_slice(ids); + padded.push(0); + let t = padded.len(); + let ids_t = Tensor::from_vec(padded, t, &self.dev).map_err(err)?; + let s_dec = style.narrow(1, 0, 128).map_err(err)?; + let s_pro = style.narrow(1, 128, 128).map_err(err)?; + let bert = self.albert.fwd(&ids_t)?; + let d_en = self.bert_enc.fwd(&bert)?.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?; // [1, 512, T] + let d = self.pred.encode(&d_en, &s_pro)?; // [1, T, 640] + let dur = self.pred.durations(&d, speed)?; + let frames: usize = dur.iter().sum(); + // alignment [T, F]: token i covers its run of frames + let mut aln = vec![0f32; t * frames]; + let mut at = 0usize; + for (i, n) in dur.iter().enumerate() { + for k in at..at + n { + if let Some(v) = aln.get_mut(i * frames + k) { + *v = 1.0; + } + } + at += n; + } + let aln = Tensor::from_vec(aln, (1, t, frames), &self.dev).map_err(err)?; + let en = d.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?.matmul(&aln).map_err(err)?; // [1, 640, F] + let (f0, n) = self.pred.f0_n(&en, &s_pro)?; + let t_en = self.text.fwd(&ids_t)?; // [1, 512, T] + let asr = t_en.matmul(&aln).map_err(err)?; // [1, 512, F] + self.dec.fwd(&asr, &f0, &n, &s_dec, seed) + } +} diff --git a/crates/tts/src/kokoro/weights.rs b/crates/tts/src/kokoro/weights.rs new file mode 100644 index 000000000..b65e8de49 --- /dev/null +++ b/crates/tts/src/kokoro/weights.rs @@ -0,0 +1,266 @@ +//! Loading the Kokoro-82M checkpoint, its config and voice packs. +//! +//! The official release is a PyTorch zip checkpoint (`kokoro-v1_0.pth`) holding five state +//! dictionaries (`bert`, `bert_encoder`, `predictor`, `text_encoder`, `decoder`), each with a +//! `module.` prefix. They are read with candle's restricted pickle reader, which only rebuilds +//! tensors and never runs code from the file. Weight-normalised layers (`weight_g`, `weight_v`) are +//! fused into a plain weight on load. Voice packs (`voices/.pt`) hold one bare float32 tensor +//! `[510, 1, 256]`, read straight from the zip archive. + +use std::collections::HashMap; +use std::io::Read; +use std::path::Path; + +use candle_core::{DType, Device, Tensor}; + +use crate::TtsError; + +pub(crate) fn err(e: impl std::fmt::Display) -> TtsError { + TtsError::Model(e.to_string()) +} + +/// The architecture this code implements; a checkpoint whose config differs is refused. +pub(crate) struct Config { + pub vocab: HashMap, +} + +/// Values of `config.json` this implementation is written for. +const EXPECT: &[(&str, &str)] = &[ + ("hidden_dim", "512"), + ("style_dim", "128"), + ("n_token", "178"), + ("max_dur", "50"), + ("n_layer", "3"), + ("text_encoder_kernel_size", "5"), + ("istftnet.gen_istft_n_fft", "20"), + ("istftnet.gen_istft_hop_size", "5"), + ("istftnet.upsample_rates", "[10,6]"), + ("istftnet.upsample_kernel_sizes", "[20,12]"), + ("istftnet.resblock_kernel_sizes", "[3,7,11]"), + ("istftnet.upsample_initial_channel", "512"), + ("plbert.hidden_size", "768"), + ("plbert.num_attention_heads", "12"), + ("plbert.num_hidden_layers", "12"), + ("plbert.intermediate_size", "2048"), +]; + +pub(crate) fn read_config(path: &Path) -> Result { + let bytes = std::fs::read(path).map_err(|e| err(format!("{}: {e}", path.display())))?; + if bytes.len() > 1 << 20 { + return Err(err("config.json is too large")); + } + let v: serde_json::Value = serde_json::from_slice(&bytes).map_err(err)?; + for (key, want) in EXPECT { + let mut cur = &v; + for part in key.split('.') { + cur = cur.get(part).ok_or_else(|| err(format!("config.json: missing `{key}`")))?; + } + let got = serde_json::to_string(cur).map_err(err)?; + if got != *want { + return Err(err(format!("config.json: `{key}` is {got}, this build supports {want}"))); + } + } + let table = v.get("vocab").and_then(|t| t.as_object()).ok_or_else(|| err("config.json: missing `vocab`"))?; + let mut vocab = HashMap::new(); + for (sym, id) in table { + let mut chars = sym.chars(); + let (Some(c), None) = (chars.next(), chars.next()) else { continue }; + let Some(id) = id.as_u64().filter(|i| *i > 0 && *i < 178) else { continue }; + vocab.insert(c, id as u32); + } + if vocab.len() < 100 { + return Err(err("config.json: the phoneme vocabulary is incomplete")); + } + Ok(Config { vocab }) +} + +/// All model tensors by full name (`decoder.generator.conv_post.weight`…), weight norm fused. +pub(crate) struct Weights { + map: HashMap, +} + +const PARTS: [&str; 5] = ["bert", "bert_encoder", "predictor", "text_encoder", "decoder"]; + +impl Weights { + pub(crate) fn load(path: &Path) -> Result { + let mut raw: HashMap = HashMap::new(); + for part in PARTS { + let tensors = candle_core::pickle::read_all_with_key(path, Some(part)).map_err(|e| err(format!("{}: {e}", path.display())))?; + if tensors.is_empty() { + return Err(err(format!("{}: no `{part}` weights", path.display()))); + } + for (name, t) in tensors { + let name = name.strip_prefix("module.").unwrap_or(&name); + let t = t.to_dtype(DType::F32).map_err(err)?; + raw.insert(format!("{part}.{name}"), t); + } + } + // fuse weight norm: w = g · v / ‖v‖ (norm over every dim but the first) + let mut map = HashMap::with_capacity(raw.len()); + let gs: Vec = raw.keys().filter(|k| k.ends_with(".weight_g")).cloned().collect(); + for g_name in gs { + let base = g_name.trim_end_matches("_g").to_string(); + let v_name = format!("{base}_v"); + let (Some(g), Some(v)) = (raw.remove(&g_name), raw.remove(&v_name)) else { + return Err(err(format!("{g_name} without {v_name}"))); + }; + let norm = v.sqr().map_err(err)?.sum_keepdim(1).map_err(err)?.sum_keepdim(2).map_err(err)?.sqrt().map_err(err)?; + let w = v.broadcast_div(&norm).map_err(err)?.broadcast_mul(&g).map_err(err)?; + map.insert(base, w); + } + map.extend(raw); + Ok(Weights { map }) + } + + pub(crate) fn get(&self, name: &str) -> Result { + self.map.get(name).cloned().ok_or_else(|| err(format!("missing weight `{name}`"))) + } + + /// `get` with an expected shape. + pub(crate) fn shaped(&self, name: &str, dims: &[usize]) -> Result { + let t = self.get(name)?; + if t.dims() != dims { + return Err(err(format!("weight `{name}` is {:?}, expected {dims:?}", t.dims()))); + } + Ok(t) + } +} + +/// Number of style vectors in a voice pack (one per input length 1..=510). +pub(crate) const PACK_LEN: usize = 510; +pub(crate) const STYLE: usize = 256; + +/// Read a voice pack: a zip whose `/data/0` holds `[510, 1, 256]` little-endian float32. +pub(crate) fn read_voice(path: &Path) -> Result, TtsError> { + let f = std::fs::File::open(path).map_err(|e| err(format!("{}: {e}", path.display())))?; + let mut z = zip::ZipArchive::new(f).map_err(|e| err(format!("{}: {e}", path.display())))?; + let name = (0..z.len()) + .filter_map(|i| z.by_index(i).ok().map(|e| e.name().to_string())) + .find(|n| n.ends_with("/data/0")) + .ok_or_else(|| err(format!("{}: not a voice pack", path.display())))?; + let order = (0..z.len()).filter_map(|i| z.by_index(i).ok().map(|e| e.name().to_string())).find(|n| n.ends_with("/byteorder")); + if let Some(o) = order { + let mut s = String::new(); + z.by_name(&o).map_err(err)?.take(16).read_to_string(&mut s).map_err(err)?; + if s.trim() != "little" { + return Err(err(format!("{}: unsupported byte order `{s}`", path.display()))); + } + } + let want = PACK_LEN * STYLE * 4; + let mut e = z.by_name(&name).map_err(err)?; + if e.size() != want as u64 { + return Err(err(format!("{}: voice data is {} bytes, expected {want}", path.display(), e.size()))); + } + let mut buf = Vec::with_capacity(want); + e.by_ref().take(want as u64).read_to_end(&mut buf).map_err(err)?; + if buf.len() != want { + return Err(err(format!("{}: truncated voice data", path.display()))); + } + let v: Vec = buf.as_chunks::<4>().0.iter().map(|c| f32::from_le_bytes(*c)).collect(); + if v.iter().any(|x| !x.is_finite()) { + return Err(err(format!("{}: voice data is not finite", path.display()))); + } + Ok(v) +} + +/// The style vector for an input of `n` phonemes: `[1, 256]`. +pub(crate) fn style_for(pack: &[f32], n: usize, dev: &Device) -> Result { + let row = n.clamp(1, PACK_LEN) - 1; + let s = pack.get(row * STYLE..(row + 1) * STYLE).ok_or_else(|| err("voice pack too short"))?; + Tensor::from_slice(s, (1, STYLE), dev).map_err(err) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + + fn tmp(name: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("filmcraft-kokoro-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + /// A voice-pack-shaped zip with `data` as `x/data/0`. + fn pack(dir: &Path, data: &[u8], order: &str) -> std::path::PathBuf { + let p = dir.join("v.pt"); + let mut z = zip::ZipWriter::new(std::fs::File::create(&p).unwrap()); + let o = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + z.start_file("x/data.pkl", o).unwrap(); + z.write_all(b"not used").unwrap(); + z.start_file("x/byteorder", o).unwrap(); + z.write_all(order.as_bytes()).unwrap(); + z.start_file("x/data/0", o).unwrap(); + z.write_all(data).unwrap(); + z.finish().unwrap(); + p + } + + fn floats(n: usize, v: f32) -> Vec { + std::iter::repeat_n(v, n).flat_map(f32::to_le_bytes).collect() + } + + #[test] + fn reads_a_well_formed_voice_pack_and_picks_the_row_by_length() { + let d = tmp("ok"); + let mut data = floats(PACK_LEN * STYLE, 0.0); + // row 4 (inputs of 5 phonemes) starts with 7.0 + data[4 * STYLE * 4..4 * STYLE * 4 + 4].copy_from_slice(&7.0f32.to_le_bytes()); + let v = read_voice(&pack(&d, &data, "little")).unwrap(); + assert_eq!(v.len(), PACK_LEN * STYLE); + let s = style_for(&v, 5, &Device::Cpu).unwrap(); + assert_eq!(s.dims(), &[1, STYLE]); + assert_eq!(s.to_vec2::().unwrap()[0][0], 7.0); + // lengths outside 1..=510 clamp instead of failing + assert!(style_for(&v, 0, &Device::Cpu).is_ok()); + assert!(style_for(&v, 100_000, &Device::Cpu).is_ok()); + let _ = std::fs::remove_dir_all(&d); + } + + #[test] + fn hostile_voice_packs_are_errors_not_panics() { + let d = tmp("bad"); + let cases: Vec<(&str, std::path::PathBuf)> = + vec![("short", pack(&d, &floats(10, 0.0), "little")), ("long", pack(&d.join(".."), &floats(PACK_LEN * STYLE + 1, 0.0), "little"))]; + for (what, p) in cases { + assert!(read_voice(&p).is_err(), "{what}"); + } + let p = pack(&d, &floats(PACK_LEN * STYLE, 0.0), "big"); + assert!(read_voice(&p).is_err(), "big endian"); + let p = pack(&d, &floats(PACK_LEN * STYLE, f32::NAN), "little"); + assert!(read_voice(&p).is_err(), "NaN"); + let junk = d.join("junk.pt"); + std::fs::write(&junk, b"PK\x03\x04 definitely not a zip").unwrap(); + assert!(read_voice(&junk).is_err()); + std::fs::write(&junk, b"").unwrap(); + assert!(read_voice(&junk).is_err()); + assert!(read_voice(&d.join("missing.pt")).is_err()); + // a checkpoint that isn't one + assert!(Weights::load(&junk).is_err()); + let _ = std::fs::remove_dir_all(&d); + } + + #[test] + fn config_must_match_the_implemented_architecture() { + let d = tmp("cfg"); + let p = d.join("config.json"); + let good = r#"{"hidden_dim":512,"style_dim":128,"n_token":178,"max_dur":50,"n_layer":3,"text_encoder_kernel_size":5, + "istftnet":{"gen_istft_n_fft":20,"gen_istft_hop_size":5,"upsample_rates":[10,6],"upsample_kernel_sizes":[20,12],"resblock_kernel_sizes":[3,7,11],"upsample_initial_channel":512}, + "plbert":{"hidden_size":768,"num_attention_heads":12,"num_hidden_layers":12,"intermediate_size":2048},"vocab":VOCAB}"#; + let vocab: String = format!("{{{}}}", (0..110).map(|i| format!("\"{}\":{}", char::from_u32(0x100 + i).unwrap(), i + 1)).collect::>().join(",")); + std::fs::write(&p, good.replace("VOCAB", &vocab)).unwrap(); + assert_eq!(read_config(&p).unwrap().vocab.len(), 110); + std::fs::write(&p, good.replace("VOCAB", &vocab).replace("\"hidden_dim\":512", "\"hidden_dim\":256")).unwrap(); + assert!(read_config(&p).err().unwrap().to_string().contains("hidden_dim")); + std::fs::write(&p, good.replace("VOCAB", "{\"a\":1}")).unwrap(); + assert!(read_config(&p).is_err(), "tiny vocabulary"); + for junk in ["", "{", "[]", "null", "{\"vocab\":5}"] { + std::fs::write(&p, junk).unwrap(); + assert!(read_config(&p).is_err(), "{junk}"); + } + std::fs::write(&p, vec![b' '; (1 << 20) + 1]).unwrap(); + assert!(read_config(&p).is_err(), "oversized"); + let _ = std::fs::remove_dir_all(&d); + } +} diff --git a/crates/tts/src/lib.rs b/crates/tts/src/lib.rs new file mode 100644 index 000000000..4a703f977 --- /dev/null +++ b/crates/tts/src/lib.rs @@ -0,0 +1,222 @@ +//! Text to speech for FilmCraft's Text to Speech panel (narrations). +//! +//! - [`Voice`]: the trait every voice implements: a script in, mono samples out. +//! - [`voices`] / [`voice`]: the voice catalogue (id, name, language, gender, engine, licence). +//! - [`script`]: splits a script into speech and pause markers (`[pause 1s]`, `[pause 500ms]`). +//! - [`formant`]: the built-in voices, an original source–filter formant synthesizer. They need +//! no download and work everywhere (also in the web build); they sound robotic. The neural voices +//! (Kokoro-82M, downloaded on first use) are added behind a feature in a later milestone. +//! +//! Everything is deterministic: the same script and settings give the same samples. + +pub mod catalog; +pub mod formant; +#[cfg(feature = "kokoro")] +pub mod kokoro; +#[cfg(feature = "kokoro")] +pub mod neural; +pub mod script; + +use serde::Serialize; + +/// Sample rate of every voice's output. +pub const SAMPLE_RATE: u32 = 24_000; +/// Longest narration a voice produces, in seconds. +pub const MAX_SECONDS: f64 = 30.0 * 60.0; +/// Longest script accepted, in bytes (same as `filmcraft_project::narration::MAX_TEXT_BYTES`). +pub const MAX_TEXT_BYTES: usize = 64 * 1024; +/// Pace limits. +pub const MIN_PACE: f64 = 0.5; +pub const MAX_PACE: f64 = 2.0; +/// Pitch shift limit in semitones (either way). +pub const MAX_SEMITONES: f64 = 12.0; + +#[derive(Debug, thiserror::Error, PartialEq)] +pub enum TtsError { + #[error("unknown voice `{0}`")] + UnknownVoice(String), + #[error("the script has nothing to say")] + Empty, + #[error("the script is longer than {} KB", MAX_TEXT_BYTES / 1024)] + TextTooLong, + #[error("the narration would be longer than {} minutes", MAX_SECONDS / 60.0)] + AudioTooLong, + #[error("{0}")] + Invalid(String), + #[error("voice model: {0}")] + Model(String), + #[error("the natural voices are not downloaded yet")] + NotInstalled, + #[error("natural voices are not available in this build (built without the `kokoro` feature)")] + Unavailable, +} + +/// How a voice speaks. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Params { + /// Pitch shift in semitones, ±[`MAX_SEMITONES`]. + pub semitones: f64, + /// Speaking rate, [`MIN_PACE`]–[`MAX_PACE`] (1 = normal). + pub pace: f64, +} + +impl Default for Params { + fn default() -> Self { + Params { semitones: 0.0, pace: 1.0 } + } +} + +impl Params { + pub fn check(&self) -> Result<(), TtsError> { + if !(MIN_PACE..=MAX_PACE).contains(&self.pace) { + return Err(TtsError::Invalid(format!("pace must be between {MIN_PACE} and {MAX_PACE}"))); + } + if !(-MAX_SEMITONES..=MAX_SEMITONES).contains(&self.semitones) { + return Err(TtsError::Invalid(format!("pitch must be within ±{MAX_SEMITONES} semitones"))); + } + Ok(()) + } +} + +/// Mono audio at [`SAMPLE_RATE`]. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Audio { + pub samples: Vec, + pub sample_rate: u32, +} + +impl Audio { + pub fn seconds(&self) -> f64 { + self.samples.len() as f64 / f64::from(self.sample_rate.max(1)) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub enum Gender { + Female, + Male, +} + +/// A voice in the catalogue. +#[derive(Clone, Copy, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct VoiceInfo { + pub id: &'static str, + pub name: &'static str, + /// BCP 47 tag. + pub language: &'static str, + pub gender: Gender, + /// `basic` (built in) or `neural` (downloaded). + pub engine: &'static str, + pub description: &'static str, + pub license: &'static str, + /// Usable now (built in, or downloaded). + pub installed: bool, +} + +/// A voice that turns a script into audio. +pub trait Voice: Send + Sync { + fn info(&self) -> VoiceInfo; + /// Speak `text` (pause markers allowed). Fails on an empty or over-long script, or bad params. + fn synthesize(&self, text: &str, params: &Params) -> Result; +} + +/// Languages offered (English only for now). +pub const LANGUAGES: &[(&str, &str)] = &[("en-US", "English (United States)")]; + +/// Every voice FilmCraft knows: the built-in voices, then the neural voices (installed when their +/// package is downloaded into `models_dir`). +pub fn voices_in(models_dir: Option<&std::path::Path>) -> Vec { + let installed = models_dir.is_some_and(catalog::installed) && cfg!(feature = "kokoro"); + formant::VOICES.iter().map(|v| v.info).chain(catalog::VOICES.iter().map(|v| VoiceInfo { installed, ..v.info })).collect() +} + +/// The built-in voices (always available). +pub fn voices() -> Vec { + formant::VOICES.iter().map(|v| v.info).collect() +} + +/// The voice with this id. +pub fn voice(id: &str) -> Result, TtsError> { + formant::VOICES + .iter() + .find(|v| v.info.id == id) + .map(|v| Box::new(formant::FormantVoice::new(*v)) as Box) + .ok_or_else(|| TtsError::UnknownVoice(id.to_string())) +} + +/// The voice `id`: built in, or (feature `kokoro`) a neural voice from the package in `models_dir`. +pub fn voice_in(id: &str, models_dir: Option<&std::path::Path>) -> Result, TtsError> { + if catalog::find(id).is_some() { + #[cfg(feature = "kokoro")] + { + let dir = models_dir.ok_or(TtsError::NotInstalled)?; + return neural::load(id, dir).map(|v| Box::new(v) as Box); + } + #[cfg(not(feature = "kokoro"))] + { + let _ = models_dir; + return Err(TtsError::Unavailable); + } + } + voice(id) +} + +/// The voice used when none is chosen. +pub fn default_voice_id() -> &'static str { + formant::VOICES.first().map(|v| v.info.id).unwrap_or("basic-female") +} + +/// The sentence "Hear this voice" plays. +pub const SAMPLE_SENTENCE: &str = "Hello! This is how I sound when I read your script."; + +/// Validate a script before synthesis: length, and that it says something. +pub fn check_text(text: &str) -> Result<(), TtsError> { + if text.len() > MAX_TEXT_BYTES { + return Err(TtsError::TextTooLong); + } + let says_something = script::parse(text).iter().any(|s| matches!(s, script::Segment::Speech(t) if t.chars().any(char::is_alphanumeric))); + if !says_something { + return Err(TtsError::Empty); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn catalogue_ids_are_unique_and_resolvable() { + let v = voices(); + assert!(v.len() >= 2); + for (i, a) in v.iter().enumerate() { + assert!(v.iter().skip(i + 1).all(|b| b.id != a.id), "duplicate {}", a.id); + assert_eq!(voice(a.id).unwrap().info().id, a.id); + } + assert_eq!(voice("nope").err(), Some(TtsError::UnknownVoice("nope".into()))); + assert!(voice(default_voice_id()).is_ok()); + } + + #[test] + fn params_are_checked() { + assert!(Params::default().check().is_ok()); + for p in [ + Params { pace: f64::NAN, ..Default::default() }, + Params { pace: 0.1, ..Default::default() }, + Params { semitones: 40.0, ..Default::default() }, + Params { semitones: f64::INFINITY, ..Default::default() }, + ] { + assert!(p.check().is_err(), "{p:?}"); + } + } + + #[test] + fn empty_scripts_are_refused() { + assert_eq!(check_text(""), Err(TtsError::Empty)); + assert_eq!(check_text(" [pause 1s] ... "), Err(TtsError::Empty)); + assert_eq!(check_text(&"a".repeat(MAX_TEXT_BYTES + 1)), Err(TtsError::TextTooLong)); + assert!(check_text("Hi").is_ok()); + } +} diff --git a/crates/tts/src/neural.rs b/crates/tts/src/neural.rs new file mode 100644 index 000000000..ca6f3298f --- /dev/null +++ b/crates/tts/src/neural.rs @@ -0,0 +1,133 @@ +//! The neural voices (feature `kokoro`): a script goes through the text front end +//! (`filmcraft-tts-text`: normalizer, CMUdict + letter-to-sound, chunks of at most 400 phonemes +//! and pause markers), each chunk through Kokoro-82M, pauses become exact silence, and the vocal +//! pitch is applied afterwards with the pitch shifter from `filmcraft-audio-dsp` (Kokoro has no +//! pitch control). Pace is Kokoro's speed. The model, the dictionary and voice packs are loaded +//! once per package directory and shared. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, OnceLock, PoisonError}; + +use filmcraft_audio_dsp::AudioEffect; +use filmcraft_tts_text::{Chunk, Lang, Lexicon, Overrides}; + +use crate::catalog; +use crate::kokoro::{KOKORO_RATE, Kokoro, VoicePack}; +use crate::{Audio, MAX_SECONDS, Params, TtsError, Voice, VoiceInfo, check_text}; + +/// Silence between sentences, ms. +const SENTENCE_GAP_MS: u32 = 120; + +struct Loaded { + dir: PathBuf, + model: Arc, + lexicon: Arc, + packs: HashMap<&'static str, Arc>, +} + +fn cache() -> &'static Mutex> { + static C: OnceLock>> = OnceLock::new(); + C.get_or_init(Default::default) +} + +/// A neural voice ready to speak. +pub struct NeuralVoice { + info: VoiceInfo, + model: Arc, + lexicon: Arc, + pack: Arc, +} + +/// Load (or reuse) the package in `models_dir` and the voice `id`. +pub fn load(id: &str, models_dir: &Path) -> Result { + let v = catalog::find(id).ok_or_else(|| TtsError::UnknownVoice(id.to_string()))?; + if !catalog::installed(models_dir) { + return Err(TtsError::NotInstalled); + } + let dir = catalog::package_dir(models_dir); + let mut guard = cache().lock().unwrap_or_else(PoisonError::into_inner); + if guard.as_ref().is_none_or(|l| l.dir != dir) { + let model = Arc::new(Kokoro::load(&dir)?); + let bytes = std::fs::read(dir.join("cmudict.dict")).map_err(|e| TtsError::Model(format!("cmudict.dict: {e}")))?; + let lexicon = Arc::new(Lexicon::parse_cmudict(&bytes).map_err(|e| TtsError::Model(format!("cmudict.dict: {e}")))?); + *guard = Some(Loaded { dir: dir.clone(), model, lexicon, packs: HashMap::new() }); + } + let loaded = guard.as_mut().ok_or_else(|| TtsError::Model("voice package not loaded".into()))?; + let pack = match loaded.packs.get(v.file) { + Some(p) => p.clone(), + None => { + let p = Arc::new(VoicePack::load(&dir.join(v.file))?); + loaded.packs.insert(v.file, p.clone()); + p + } + }; + Ok(NeuralVoice { info: VoiceInfo { installed: true, ..v.info }, model: loaded.model.clone(), lexicon: loaded.lexicon.clone(), pack }) +} + +impl Voice for NeuralVoice { + fn info(&self) -> VoiceInfo { + self.info + } + + fn synthesize(&self, text: &str, params: &Params) -> Result { + check_text(text)?; + params.check()?; + let chunks = filmcraft_tts_text::phonemize(text, Lang::EnUs, &self.lexicon, &Overrides::default()).map_err(|e| TtsError::Invalid(e.to_string()))?; + let rate = KOKORO_RATE; + let max = (MAX_SECONDS * f64::from(rate)) as usize; + let mut out: Vec = Vec::new(); + let mut spoke = false; + for (i, c) in chunks.iter().enumerate() { + match c { + Chunk::Pause { millis } => { + let n = (u64::from(*millis) * u64::from(rate) / 1000) as usize; + out.resize(out.len().saturating_add(n), 0.0); + } + Chunk::Phonemes(p) => { + if self.model.ids(p).is_empty() { + continue; + } + if spoke { + out.resize(out.len().saturating_add((SENTENCE_GAP_MS * rate / 1000) as usize), 0.0); + } + let s = self.model.speak(p, &self.pack, params.pace, i as u64 + 1)?; + out.extend_from_slice(&s); + spoke = true; + } + } + if out.len() > max { + return Err(TtsError::AudioTooLong); + } + } + if !spoke { + return Err(TtsError::Empty); + } + if params.semitones.abs() > 1e-6 { + out = pitch_shift(out, params.semitones, rate); + } + // headroom: never clip + let peak = out.iter().fold(0f32, |m, x| m.max(x.abs())); + if peak > 0.98 { + let g = 0.98 / peak; + out.iter_mut().for_each(|x| *x *= g); + } + Ok(Audio { samples: out, sample_rate: rate }) + } +} + +/// Shift pitch by `semitones` keeping the length (latency compensated). +fn pitch_shift(mut x: Vec, semitones: f64, rate: u32) -> Vec { + let mut ps = filmcraft_audio_dsp::effects::PitchShifter::new(rate as f32, 1); + ps.set_param("semitones", semitones as f32); + ps.reset(); + let lat = ps.latency(); + let n = x.len(); + x.resize(n + lat, 0.0); + for block in x.chunks_mut(4096) { + ps.process(&mut [block]); + } + x.drain(..lat.min(x.len())); + x.truncate(n); + x +} diff --git a/crates/tts/src/script.rs b/crates/tts/src/script.rs new file mode 100644 index 000000000..19effcf6d --- /dev/null +++ b/crates/tts/src/script.rs @@ -0,0 +1,105 @@ +//! Script parsing: speech text and pause markers. +//! +//! A pause marker is `[pause s]` or `[pause ms]` (case-insensitive, `n` a decimal number, +//! spaces optional: `[pause 1.5s]`, `[Pause 500 ms]`). Pauses are capped at [`MAX_PAUSE_MS`]. A +//! bracket that is not a well-formed marker is ordinary text. The Text to Speech panel's +//! *Add pause* button inserts `[pause 1s]`. + +/// Longest single pause. +pub const MAX_PAUSE_MS: u32 = 10_000; + +/// A piece of a script. +#[derive(Clone, Debug, PartialEq)] +pub enum Segment { + Speech(String), + Pause { millis: u32 }, +} + +/// The marker *Add pause* inserts. +pub const PAUSE_MARKER: &str = "[pause 1s]"; + +/// Split `text` into speech and pauses, in order. Empty speech pieces are dropped. +pub fn parse(text: &str) -> Vec { + let mut out = Vec::new(); + let mut speech = String::new(); + let mut rest = text; + while let Some(open) = rest.find('[') { + let (before, from) = rest.split_at(open); + speech.push_str(before); + let close = from.find(']'); + match (close, close.and_then(|c| from.get(..=c)).and_then(pause_millis)) { + (Some(close), Some(millis)) => { + flush(&mut out, &mut speech); + out.push(Segment::Pause { millis }); + rest = from.get(close + 1..).unwrap_or(""); + } + _ => { + speech.push('['); + rest = from.get(1..).unwrap_or(""); + } + } + } + speech.push_str(rest); + flush(&mut out, &mut speech); + out +} + +fn flush(out: &mut Vec, speech: &mut String) { + if !speech.trim().is_empty() { + out.push(Segment::Speech(std::mem::take(speech))); + } + speech.clear(); +} + +/// `[pause 1.5s]` → 1500. None if `marker` is not a pause marker. +fn pause_millis(marker: &str) -> Option { + let inner = marker.strip_prefix('[')?.strip_suffix(']')?.trim(); + let lower = inner.to_ascii_lowercase(); + let arg = lower.strip_prefix("pause")?.trim(); + let (num, scale) = if let Some(n) = arg.strip_suffix("ms") { (n.trim(), 1.0) } else { (arg.strip_suffix('s')?.trim(), 1000.0) }; + if num.is_empty() || num.len() > 12 || !num.chars().all(|c| c.is_ascii_digit() || c == '.') { + return None; + } + let v: f64 = num.parse().ok()?; + let ms = (v * scale).round(); + if !ms.is_finite() || ms < 0.0 { + return None; + } + Some(ms.min(f64::from(MAX_PAUSE_MS)) as u32) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sp(s: &str) -> Segment { + Segment::Speech(s.into()) + } + + #[test] + fn splits_speech_and_pauses() { + assert_eq!(parse("Hello. [pause 1s] World"), vec![sp("Hello. "), Segment::Pause { millis: 1000 }, sp(" World")]); + assert_eq!(parse("[Pause 500 ms]Go"), vec![Segment::Pause { millis: 500 }, sp("Go")]); + assert_eq!(parse("a[pause 1.5s][pause 2s]b"), vec![sp("a"), Segment::Pause { millis: 1500 }, Segment::Pause { millis: 2000 }, sp("b")]); + assert_eq!(parse(PAUSE_MARKER), vec![Segment::Pause { millis: 1000 }]); + } + + #[test] + fn malformed_markers_are_text() { + assert_eq!(parse("[pause]"), vec![sp("[pause]")]); + assert_eq!(parse("[pause 1x] a"), vec![sp("[pause 1x] a")]); + assert_eq!(parse("[pause 1s"), vec![sp("[pause 1s")]); + assert_eq!(parse("[note] [pause -1s]"), vec![sp("[note] [pause -1s]")]); + assert_eq!(parse("[[pause 1s]]"), vec![sp("["), Segment::Pause { millis: 1000 }, sp("]")]); + } + + #[test] + fn hostile_inputs_never_panic_and_pauses_are_capped() { + assert_eq!(parse("[pause 999999999999s]"), vec![Segment::Pause { millis: MAX_PAUSE_MS }]); + assert_eq!(parse("[pause 1e9s]"), vec![sp("[pause 1e9s]")]); + for s in ["", "[", "]", "[[[[", "日本語[pause 1s]👍", "\u{301}[", "[pause 1.2.3s]", "\0[pause 0s]"] { + let _ = parse(s); + } + assert_eq!(parse(" "), vec![]); + } +} diff --git a/crates/tts/tests/data/sentences.txt b/crates/tts/tests/data/sentences.txt new file mode 100644 index 000000000..a77d27781 --- /dev/null +++ b/crates/tts/tests/data/sentences.txt @@ -0,0 +1,50 @@ +Welcome to the project list, where every job you create is stored. +Click the blue button at the top of the page to start a new project. +The filter menu lets you sort projects by date, by name, or by owner. +Each card shows when the project was created and when it was last modified. +Our team finished the review on Tuesday morning. +The meeting starts at 3:30 p.m. in the main conference room. +It costs $3.50 to ship each sample. +Sales grew by 20% over the last quarter. +The building was finished in 1999 and renovated in 2015. +Please call 555-0134 if you have any questions. +The report is due on October 7th. +She bought 12 apples, 6 oranges, and a bag of grapes. +Doctor Smith lives on Main Street near the library. +The NASA mission launched before sunrise. +The TDA tool exports every diagram as a PDF. +Turn left at the second light and park behind the bank. +A quiet river runs through the center of the old town. +The children played in the garden until it got dark. +Remember to save your work before you close the window. +This video will show you how to edit a narration clip. +The weather tomorrow will be sunny with a light breeze. +He read the book in a single afternoon. +The price dropped to $1,250 after the holiday sale. +Our office opens at nine and closes at five. +The first chapter explains the basic ideas. +Water boils at one hundred degrees Celsius. +They traveled by train from the coast to the mountains. +The software update fixes several small problems. +Every frame is rendered in less than a second. +Choose a voice, type your script, and press save. +The museum is closed on Mondays and public holidays. +A good narrator speaks slowly and clearly. +The new library has more than ten thousand books. +Write down the steps so that anyone can repeat them. +The engineer checked every valve before the test. +Our company was founded in 1987 in a small garage. +The second version runs twice as fast as the first. +Bring a jacket, because the evening will be cold. +The camera follows the bird as it flies over the lake. +Thank you for watching, and see you in the next video. +The orchestra played for almost two hours. +Most users finish the setup in about five minutes. +The road was closed because of the heavy snow. +Please confirm your email address to continue. +The garden is full of roses, tulips, and sunflowers. +Our goal is to make editing simple for everyone. +The pilot announced that we would land early. +Fresh bread is baked here every morning. +The team celebrated the launch with a small party. +That is everything you need to know to get started. diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index d91bb7e51..4a99b17b2 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -16,6 +16,7 @@ filmcraft-color = { workspace = true } filmcraft-frame = { workspace = true } filmcraft-media = { workspace = true } filmcraft-project = { workspace = true } +filmcraft-tts = { workspace = true } filmcraft-edit = { workspace = true } filmcraft-render = { workspace = true } filmcraft-text = { workspace = true } diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index 423a3fffb..162602581 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -10,6 +10,7 @@ //! - `ui.elements {prefix?}`: registered widgets (optionally filtered by id prefix) //! - `ui.set {tool?, workspace?, mode?, theme?, focused?, playbackRes?, timeline?:{pps,scroll}}` //! - `ui.panel.show {panel}` / `ui.panel.close {panel}` +//! - `ui.set {tts: {voice, pitch, pace, text…}}`: patch the Text to Speech panel draft //! - `ui.click {id | x,y, button?, count?, modifiers?}` / `ui.move {x,y}` / `ui.scroll {x,y,dx,dy}` //! - `ui.drag {from:{id|x,y}, to:{id|x,y}, steps?, modifiers?}`: synthetic press-move-release //! - `ui.key {key, command?, shift?, alt?, ctrl?}` / `ui.type {text}` @@ -123,6 +124,15 @@ pub fn handle(app: &mut FilmcraftApp, ctx: &egui::Context, req: &ControlRequest) ok(serde_json::to_value(app.auto.query(prefix)).unwrap_or_default()) } "ui.set" => { + // Text to Speech panel draft (voice, pitch, pace, text…), merged into the current one + if let Some(patch) = p.get("tts") { + let mut cur = serde_json::to_value(&app.ui.tts).unwrap_or_default(); + merge(&mut cur, patch); + match serde_json::from_value(cur) { + Ok(d) => app.ui.tts = d, + Err(e) => return err(format!("bad `tts`: {e}")), + } + } if let Some(t) = s("tool") { match Tool::from_name(t) { Some(t) => app.ui.tool = t, diff --git a/crates/ui-egui/src/dock.rs b/crates/ui-egui/src/dock.rs index f98036e49..05acf45b7 100644 --- a/crates/ui-egui/src/dock.rs +++ b/crates/ui-egui/src/dock.rs @@ -30,6 +30,7 @@ pub enum PanelKind { LumetriScopes, EssentialGraphics, EssentialSound, + TextToSpeech, Properties, Text, Events, @@ -39,7 +40,7 @@ pub enum PanelKind { } impl PanelKind { - pub const ALL: [PanelKind; 26] = [ + pub const ALL: [PanelKind; 27] = [ PanelKind::Project, PanelKind::MediaBrowser, PanelKind::Libraries, @@ -60,6 +61,7 @@ impl PanelKind { PanelKind::LumetriScopes, PanelKind::EssentialGraphics, PanelKind::EssentialSound, + PanelKind::TextToSpeech, PanelKind::Properties, PanelKind::Text, PanelKind::Events, @@ -89,6 +91,7 @@ impl PanelKind { PanelKind::LumetriScopes => "Lumetri Scopes", PanelKind::EssentialGraphics => "Essential Graphics", PanelKind::EssentialSound => "Essential Sound", + PanelKind::TextToSpeech => "Text to Speech", PanelKind::Properties => "Properties", PanelKind::Text => "Text", PanelKind::Events => "Events", @@ -268,7 +271,7 @@ pub fn workspace(name: &str) -> DockNode { hsplit(Ratio(0.5), tabs(&[AudioTrackMixer, Source, AudioClipMixer, EffectControls], 0), tabs(&[Program], 0)), hsplit(Ratio(0.3), tabs(&[Project, MediaBrowser, Effects, Markers, History], 0), bottom_editing()), ), - tabs(&[EssentialSound], 0), + tabs(&[EssentialSound, TextToSpeech], 0), ), "Captions and Graphics" => hsplit( FixedB(330.0), @@ -284,7 +287,7 @@ pub fn workspace(name: &str) -> DockNode { hsplit( Ratio(0.5), tabs(&[Source, EffectControls, AudioClipMixer, Metadata, LumetriScopes, Text], 0), - hsplit(Ratio(0.6), tabs(&[Program], 0), tabs(&[LumetriColor, EssentialGraphics, EssentialSound, Properties, AudioTrackMixer], 0)), + hsplit(Ratio(0.6), tabs(&[Program], 0), tabs(&[LumetriColor, EssentialGraphics, EssentialSound, TextToSpeech, Properties, AudioTrackMixer], 0)), ), hsplit(Ratio(0.26), tabs(&[Project, MediaBrowser, Libraries, Info, Effects, Markers, History], 0), bottom_editing()), ), diff --git a/crates/ui-egui/src/i18n/es.tsv b/crates/ui-egui/src/i18n/es.tsv index ed023afda..08f899ecb 100644 --- a/crates/ui-egui/src/i18n/es.tsv +++ b/crates/ui-egui/src/i18n/es.tsv @@ -1253,6 +1253,9 @@ Active Monitor Monitor activo Add Añadir Add tracks automatically when editing source clips onto the Timeline Añadir pistas automáticamente al editar clips de origen en la línea de tiempo + Place files dropped onto the Timeline directly on the Timeline Colocar los archivos soltados en la línea de tiempo directamente en ella + 1 file 1 archivo + {n} files {n} archivos Additional Scale Escala adicional Additive Dissolve Disolución aditiva Additive Dissolve (Legacy) Disolución aditiva (heredada) @@ -3488,6 +3491,8 @@ New Sequence Nueva secuencia Tracks Pistas tracks pistas + Clear Transition Borrar transición + Set Transition Duration Establecer duración de la transición # System appearance modes and theme cards. Active Activo @@ -3505,3 +3510,5 @@ Solid Sólido GPU Rendering Renderizado por GPU Composite on the GPU when available Componer en la GPU cuando esté disponible + Text to Speech Texto a voz + Edit Narration Editar narración diff --git a/crates/ui-egui/src/i18n/ja.tsv b/crates/ui-egui/src/i18n/ja.tsv index 73f23374d..8dbf44199 100644 --- a/crates/ui-egui/src/i18n/ja.tsv +++ b/crates/ui-egui/src/i18n/ja.tsv @@ -1253,6 +1253,9 @@ Active Monitor アクティブなモニター Add 加算 Add tracks automatically when editing source clips onto the Timeline ソースクリップをタイムラインに配置するとき、トラックを自動的に追加する + Place files dropped onto the Timeline directly on the Timeline タイムラインにドロップしたファイルをタイムラインに直接配置 + 1 file 1 個のファイル + {n} files {n} 個のファイル Additional Scale 追加スケール Additive Dissolve 加算ディゾルブ Additive Dissolve (Legacy) 加算ディゾルブ (従来版) @@ -3498,3 +3501,7 @@ Solid べた塗り GPU Rendering GPU レンダリング Composite on the GPU when available 利用可能な場合は GPU で合成 + Text to Speech テキスト読み上げ + Edit Narration ナレーションを編集 + Clear Transition トランジションを消去 + Set Transition Duration トランジションのデュレーションを設定 diff --git a/crates/ui-egui/src/i18n/zh-cn.tsv b/crates/ui-egui/src/i18n/zh-cn.tsv index 159acb113..731067f98 100644 --- a/crates/ui-egui/src/i18n/zh-cn.tsv +++ b/crates/ui-egui/src/i18n/zh-cn.tsv @@ -1246,6 +1246,9 @@ Active Monitor 活动监视器 Add 添加 Add tracks automatically when editing source clips onto the Timeline 将源剪辑编辑到时间轴时自动添加轨道 + Place files dropped onto the Timeline directly on the Timeline 将拖放到时间轴上的文件直接放置在时间轴上 + 1 file 1 个文件 + {n} files {n} 个文件 Additional Scale 附加缩放 Additive Dissolve 添加式溶解 Additive Dissolve (Legacy) 添加式溶解（旧版） @@ -3487,3 +3490,7 @@ Sync with system 跟随系统 GPU Rendering GPU 渲染 Composite on the GPU when available 可用时在 GPU 上合成 + Text to Speech 文字转语音 + Edit Narration 编辑旁白 + Clear Transition 清除过渡 + Set Transition Duration 设置过渡持续时间 diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 82cf9317c..857f42a56 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -88,6 +88,11 @@ pub trait AudioOut { /// effect at the next `start`; `document_rate` is the sequence sample rate for "Attempt to /// force hardware to document sample rate". fn configure(&mut self, _hw: &filmcraft_engine::settings::AudioHardwarePrefs, _document_rate: Option) {} + /// A hardware note for the status bar — e.g. the default output device could not be opened + /// and a fallback device is in use, or no output device was found — or None. + fn note(&self) -> Option { + None + } } /// What the platform audio layer can open (Settings ▸ Audio Hardware). @@ -105,6 +110,8 @@ pub struct AudioDevices { #[derive(Default)] pub struct HostHooks { pub pick_files: Option Vec>>, + /// The cursor in physical screen pixels. Where the windowing layer reports no pointer while files are dragged in from the OS (Windows), a drop is placed where this says it landed. + pub cursor_screen_position: Option Option<(i32, i32)>>>, pub pick_save: Option Option>>, pub pick_open_project: Option Option>>, /// Save dialog with a filter: (filter name, extensions, suggested file name) → path. @@ -170,6 +177,8 @@ pub enum Dialog { AddTracks, /// Sequence ▸ Sequence Settings…. SequenceSettings, + /// Set Transition Duration (double-click a transition in the Timeline). + TransitionDuration, } #[derive(Default)] @@ -277,6 +286,10 @@ pub struct FilmcraftApp { /// The sequence whose view `ui.timeline` holds, and that view as it was last exchanged with /// `session.state.timeline_views` (see `sync_timeline_view`). timeline_view_of: Option, + /// Items imported by an OS file drop, to place on the Timeline at the drop point this frame. + pending_timeline_drop: Option<(Vec, egui::Pos2)>, + /// Files being dragged over the window from the OS (count, pointer): the Timeline previews where they would land. + file_drag_hover: Option<(usize, egui::Pos2)>, timeline_view_last: Option, pub fps: f32, last_time: f64, @@ -465,6 +478,8 @@ impl FilmcraftApp { integrated_titlebar: false, last_timeline_width: 1000.0, timeline_view_of: None, + pending_timeline_drop: None, + file_drag_hover: None, timeline_view_last: None, fps: 60.0, last_time: 0.0, @@ -597,6 +612,9 @@ impl FilmcraftApp { self.start_audio(); } } + if let Some(note) = self.audio.as_ref().and_then(|a| a.note()) { + self.ui.status = note; + } if prev.is_none() && !self.workspace_restored { // reopen the workspace in use when the app last closed self.workspace_restored = true; @@ -695,6 +713,11 @@ impl FilmcraftApp { } if !self.ui.dock.contains(p) { let near = match p { + // a side panel beside Properties (or Essential Sound), never over the Program monitor + PanelKind::TextToSpeech => [PanelKind::Properties, PanelKind::EssentialSound, PanelKind::EssentialGraphics] + .into_iter() + .find(|k| self.ui.dock.contains(*k)) + .unwrap_or(PanelKind::Program), PanelKind::LumetriColor | PanelKind::EssentialGraphics | PanelKind::EssentialSound | PanelKind::Properties => PanelKind::Program, PanelKind::Source | PanelKind::EffectControls @@ -814,6 +837,34 @@ impl FilmcraftApp { panels::voiceover::on_play(self); } + /// Play the last Text to Speech preview (`Session::tts_preview`) on the audio output, stopping + /// timeline playback first. Starting playback again takes the output back. + pub fn play_tts_preview(&mut self) -> Result { + let audio = self.session.tts_preview.clone().ok_or("nothing to preview")?; + if self.playback.playing { + self.stop(); + } + let a = self.audio.as_mut().ok_or("no audio output (check Settings ▸ Audio Hardware)")?; + let sr = a.sample_rate().max(1); + a.stop(); + let step = f64::from(audio.sample_rate.max(1)) / f64::from(sr); + let seconds = audio.seconds(); + let mut pos = 0.0f64; + let fill = Box::new(move |buf: &mut [f32], ch: usize| { + for frame in buf.chunks_mut(ch.max(1)) { + let i = pos as usize; + let f = (pos - i as f64) as f32; + let x0 = audio.samples.get(i).copied().unwrap_or(0.0); + let x1 = audio.samples.get(i + 1).copied().unwrap_or(0.0); + frame.fill(x0 + (x1 - x0) * f); + pos += step; + } + }); + a.start(fill)?; + self.playback.audio_clock = false; + Ok(seconds) + } + pub fn stop(&mut self) { self.playback.playing = false; self.playback.stop_at = None; @@ -862,7 +913,12 @@ impl FilmcraftApp { }) }; match a.start(fill) { - Ok(_) => self.playback.audio_clock = true, + Ok(_) => { + self.playback.audio_clock = true; + if let Some(note) = self.audio.as_ref().and_then(|audio| audio.note()) { + self.ui.status = note; + } + } Err(e) => { log::warn!("audio output unavailable: {e}"); self.playback.audio_clock = false; @@ -1162,8 +1218,17 @@ impl FilmcraftApp { r } - /// Import dropped files. + /// Import dropped files. With Timeline ▸ drop imports to the timeline, the new items are also + /// queued to be placed at the pointer (see `timeline::interact`). fn handle_drops(&mut self, ctx: &egui::Context) { + // While files hover, the OS sends no cursor events: poll the cursor each frame to draw the Timeline preview. + let hovering = ctx.input(|i| i.raw.hovered_files.len()); + self.file_drag_hover = if hovering > 0 && self.session.prefs.timeline.drop_import_to_timeline { + ctx.request_repaint(); + self.drop_pointer(ctx).map(|p| (hovering, p)) + } else { + None + }; let dropped = ctx.input(|i| i.raw.dropped_files.clone()); let mut paths = Vec::new(); for f in dropped { @@ -1173,10 +1238,33 @@ impl FilmcraftApp { } } if !paths.is_empty() { - let _ = self.session.execute("file.import", json!({"paths": paths, "bin": self.import_bin().0})); + let pointer = self.drop_pointer(ctx); + let imported = self.session.execute("file.import", json!({"paths": paths, "bin": self.import_bin().0})); + if let Ok(v) = imported + && let Some(pos) = pointer + && self.session.prefs.timeline.drop_import_to_timeline + { + let items: Vec = + v["items"].as_array().map(|a| a.iter().filter_map(|i| i.as_u64()).map(filmcraft_engine::project::ItemId).collect()).unwrap_or_default(); + if !items.is_empty() { + self.pending_timeline_drop = Some((items, pos)); + } + } } } + /// Where files dragged in from the OS are over the window. The OS reports no cursor motion during such a drag + /// (winit, Windows), so ask it through the host hook, else use the last pointer position egui saw. + fn drop_pointer(&self, ctx: &egui::Context) -> Option { + let os_pointer = self + .hooks + .cursor_screen_position + .as_ref() + .and_then(|f| f()) + .and_then(|px| ctx.input(|i| i.viewport().inner_rect.map(|r| screen_px_to_ui(px, i.pixels_per_point, r.min)))); + os_pointer.or_else(|| ctx.input(|i| i.pointer.hover_pos().or(i.pointer.latest_pos()))) + } + // ---------------------------------------------------------------- input fn handle_shortcuts(&mut self, ctx: &egui::Context) { @@ -1388,6 +1476,11 @@ impl FilmcraftApp { self.advance_playback(&ctx); self.advance_source_playback(&ctx); self.scrub_audio(&ctx); + // Text to Speech: a selected narration clip loads into the panel + panels::tts::follow_selection(self); + if self.ui.tts.pending.is_some() || self.ui.tts.download_job.is_some() { + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } let t = self.tokens; let full = ui.max_rect(); ui.painter().rect_filled(full, 0.0, t.app_bg); @@ -1403,6 +1496,8 @@ impl FilmcraftApp { state::Mode::Import => panels::import_mode::show(self, ui, body), state::Mode::Export => panels::export_mode::show(self, ui, body), } + // the Timeline has had its chance to place a file drop this frame; anything left stays a bin import + self.pending_timeline_drop = None; panels::dialogs::show(self, &ctx); if !self.ui.show_status_bar { // no bar to draw the job in, but a finished preview render still plays @@ -1951,3 +2046,31 @@ mod audio_recovery_tests { app.stop(); } } + +/// A physical-pixel screen position as a point in the window's UI space, given the window's content origin in points. +/// +/// Assumes the whole virtual screen uses the window's `pixels_per_point`: with monitors of mixed +/// DPI the point can be off when the cursor and the window's origin are on different monitors. +fn screen_px_to_ui(px: (i32, i32), pixels_per_point: f32, content_min: egui::Pos2) -> egui::Pos2 { + let ppp = if pixels_per_point.is_finite() && pixels_per_point > 0.0 { pixels_per_point } else { 1.0 }; + egui::pos2(px.0 as f32 / ppp - content_min.x, px.1 as f32 / ppp - content_min.y) +} + +#[cfg(test)] +mod screen_px_tests { + use super::screen_px_to_ui; + + #[test] + fn maps_screen_pixels_into_the_window() { + let p = screen_px_to_ui((1300, 900), 2.0, egui::pos2(100.0, 50.0)); + assert_eq!((p.x, p.y), (550.0, 400.0)); + } + + #[test] + fn a_hostile_scale_does_not_divide_by_zero() { + for ppp in [0.0, -1.0, f32::NAN, f32::INFINITY] { + let p = screen_px_to_ui((10, 20), ppp, egui::pos2(0.0, 0.0)); + assert!(p.x.is_finite() && p.y.is_finite()); + } + } +} diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index e82e2c3cf..eed8fd28f 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -67,6 +67,12 @@ pub fn show(app: &mut FilmcraftApp, ctx: &egui::Context) { } return; } + Dialog::TransitionDuration => { + if !crate::panels::transition_controls::duration_dialog(app, ctx) && app.dialog == Some(d) { + app.dialog = None; + } + return; + } Dialog::SequenceSettings => { if !crate::panels::sequence_settings::show(app, ctx) && app.dialog == Some(d) { app.dialog = None; diff --git a/crates/ui-egui/src/panels/effect_controls.rs b/crates/ui-egui/src/panels/effect_controls.rs index b4142ca45..1eeb87b20 100644 --- a/crates/ui-egui/src/panels/effect_controls.rs +++ b/crates/ui-egui/src/panels/effect_controls.rs @@ -28,6 +28,19 @@ fn timeline_time_of(it: &TrackItem, m: Tick) -> Tick { it.start + Tick(((m - it.source_in).0 as f64 / it.speed.abs().max(1e-6)) as i64) } +// The press frame can include movement before the button went down. Start from the press +// origin so that only gesture motion enters the preview, including same-frame movement. +fn graph_drag_offset(ui: &egui::Ui, response: &egui::Response, previous: egui::Vec2) -> egui::Vec2 { + if response.drag_started() { + response + .interact_pointer_pos() + .zip(ui.input(|i| i.pointer.press_origin())) + .map_or_else(|| response.drag_delta(), |(position, origin)| position - origin) + } else { + previous + response.drag_delta() + } +} + fn selected_clip(app: &FilmcraftApp) -> Option<(ClipId, TrackItem, TrackKind)> { selected_clips(app).into_iter().next() } @@ -59,6 +72,11 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { let t = app.tokens; let clips = selected_clips(app); let Some((clip, it, _)) = clips.first().cloned() else { + // a transition clicked in the Timeline (#430) + if let Some(id) = crate::panels::transition_controls::selected(app) { + crate::panels::transition_controls::effect_controls(app, ui, rect, id); + return; + } crate::dock::placeholder(ui, rect, &t, tl!("(no clip selected)")); return; }; @@ -979,7 +997,7 @@ pub(crate) fn graph_rows( p.circle_filled(hp, 3.5, t.keyframe_handle); let hr = ui.interact(Rect::from_center_size(hp, vec2(10.0, 10.0)), hid.with("h"), Sense::drag()); if hr.dragged() { - let nx = hdx + hr.drag_delta().x; + let nx = graph_drag_offset(ui, &hr, vec2(hdx, 0.0)).x; ui.data_mut(|d| d.insert_temp(hid, nx)); } if hr.drag_stopped() { @@ -991,7 +1009,7 @@ pub(crate) fn graph_rows( } p.circle_filled(c, 4.5, if resp.dragged() { t.hot_text } else { t.plot_handle }); if resp.dragged() { - let ny = dy + resp.drag_delta().y; + let ny = graph_drag_offset(ui, &resp, vec2(0.0, dy)).y; ui.data_mut(|d| d.insert_temp(id, ny)); p.text(c + vec2(8.0, -10.0), Align2::LEFT_BOTTOM, format!("{:.dec$}", v_of(c.y)), Tokens::ui(10.5), t.hot_text); } @@ -1172,3 +1190,134 @@ mod lane_tests { assert_eq!(labels, ["00:01:00:00", "00:01:02:00", "00:01:04:00"]); } } + +#[cfg(test)] +mod graph_drag_tests { + use super::*; + use filmcraft_project::{Interpolation, Keyframe, Param}; + + struct Driver { + app: FilmcraftApp, + ctx: egui::Context, + clip: ClipId, + time: f64, + } + + impl Driver { + fn new() -> Self { + let mut session = filmcraft_engine::Session::default(); + session.execute("file.openDemoProject", json!({})).unwrap(); + let seq = session.state.active_sequence.unwrap(); + let clip = session.active_sequence().unwrap().video_tracks[0].items[0].id; + let it = std::sync::Arc::make_mut(&mut session.project).sequence_mut(seq).unwrap().find_item_mut(clip).unwrap().1; + let mut a = Keyframe::new(it.source_in + Tick(it.duration.0 / 4), ParamValue::Float(40.0)); + let mut b = Keyframe::new(it.source_in + Tick(it.duration.0 * 3 / 4), ParamValue::Float(80.0)); + a.interp = Interpolation::Bezier; + b.interp = Interpolation::Bezier; + it.effects + .iter_mut() + .find(|e| e.effect == "motion") + .unwrap() + .params + .insert("scale".into(), Param { value: ParamValue::Float(40.0), keyframes: vec![a, b] }); + let app = FilmcraftApp::new(session); + let ctx = egui::Context::default(); + crate::theme::install(&ctx, &app.tokens); + Self { app, ctx, clip, time: 0.0 } + } + + fn param(&self) -> Param { + self.app.session.active_sequence().unwrap().find_item(self.clip).unwrap().1.effect("motion").unwrap().params["scale"].clone() + } + + fn undo_len(&mut self) -> usize { + self.app.session.execute("history.list", json!({})).unwrap()["undo"].as_array().unwrap().len() + } + + // Draw the actual scalar graph and execute its emitted command, as the panel does. + // Current paint shapes give a real held-frame position rather than previous telemetry. + fn frame(&mut self, events: Vec) -> [Pos2; 2] { + self.time += 0.1; + self.app.auto.begin_frame(); + let mut out = self.ctx.run_ui(egui::RawInput { time: Some(self.time), events, ..Default::default() }, |ui| { + let it = self.app.session.active_sequence().unwrap().find_item(self.clip).unwrap().1.clone(); + let index = it.effects.iter().position(|e| e.effect == "motion").unwrap(); + let effect = &it.effects[index]; + let pd = effect.def().unwrap().param("scale").unwrap(); + let body = Rect::from_min_size(pos2(8.0, 8.0), vec2(320.0, 200.0)); + let lane = Rect::from_min_max(pos2(360.0, 8.0), pos2(760.0, 220.0)); + let mut actions = Vec::new(); + graph_rows(&mut self.app, ui, body, self.clip, index, None, pd, &effect.params["scale"], &lane, &it, &mut actions); + run(&mut self.app, ui.ctx(), actions); + }); + out.textures_delta.clear(); + fn first_circle(shape: &egui::Shape, radius: f32) -> Option { + match shape { + egui::Shape::Circle(circle) if circle.radius == radius => Some(circle.center), + egui::Shape::Vec(shapes) => shapes.iter().find_map(|shape| first_circle(shape, radius)), + _ => None, + } + } + [4.5, 3.5].map(|radius| { + out.shapes.iter().find_map(|shape| first_circle(&shape.shape, radius)).expect("scalar keyframe and outgoing influence are painted") + }) + } + } + + fn button(pos: Pos2, pressed: bool) -> egui::Event { + egui::Event::PointerButton { pos, button: egui::PointerButton::Primary, pressed, modifiers: Default::default() } + } + + #[test] + fn scalar_graph_press_starts_at_the_pointer_origin_and_release_is_one_undo_step() { + for influence in [false, true] { + for batched in [true, false] { + let mut d = Driver::new(); + let before = d.param(); + let undo = d.undo_len(); + let target = usize::from(influence); + let start = d.frame(vec![])[target]; + // A different prior cursor makes the pre-press movement observable. + d.frame(vec![egui::Event::PointerMoved(start - vec2(80.0, 40.0))]); + if batched { + d.frame(vec![egui::Event::PointerMoved(start), button(start, true)]); + } else { + d.frame(vec![egui::Event::PointerMoved(start)]); + d.frame(vec![button(start, true)]); + } + let pressed = d.frame(vec![])[target]; + assert!(pressed.distance(start) < 0.05, "press cannot jump: influence={influence}, batched={batched}, start={start:?}, pressed={pressed:?}"); + assert_eq!(d.param(), before); + assert_eq!(d.undo_len(), undo); + let offset = if influence { vec2(20.0, 0.0) } else { vec2(0.0, -20.0) }; + d.frame(vec![egui::Event::PointerMoved(start + offset)]); + let preview = d.frame(vec![])[target]; + assert!( + preview.distance(start + offset) < 0.05, + "preview follows only gesture motion: influence={influence}, batched={batched}, start={start:?}, preview={preview:?}" + ); + assert_eq!(d.param(), before, "held preview cannot commit"); + assert_eq!(d.undo_len(), undo); + d.frame(vec![button(start + offset, false)]); + d.frame(vec![]); + let changed = d.param(); + assert_eq!(d.undo_len(), undo + 1); + assert_eq!(changed.keyframes[0].time, before.keyframes[0].time); + assert_eq!(changed.keyframes[1], before.keyframes[1]); + if influence { + assert_eq!(changed.keyframes[0].value, before.keyframes[0].value); + assert!(changed.keyframes[0].out_influence > before.keyframes[0].out_influence); + } else { + assert!(changed.keyframes[0].value.as_f64().unwrap() > before.keyframes[0].value.as_f64().unwrap()); + assert_eq!(changed.keyframes[0].out_influence, before.keyframes[0].out_influence); + } + d.app.session.undo().expect("one graph gesture to undo"); + assert_eq!(d.param(), before); + assert_eq!(d.undo_len(), undo); + d.app.session.redo().expect("the graph gesture to redo"); + assert_eq!(d.param(), changed); + assert_eq!(d.undo_len(), undo + 1); + } + } + } +} diff --git a/crates/ui-egui/src/panels/graphics.rs b/crates/ui-egui/src/panels/graphics.rs index f424ee2bc..09aead4f5 100644 --- a/crates/ui-egui/src/panels/graphics.rs +++ b/crates/ui-egui/src/panels/graphics.rs @@ -1594,7 +1594,17 @@ pub fn properties(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { for (id, r, label) in autos { app.auto.add(&id, r, &label); } - for (cmd, p) in actions { + for (cmd, mut p) in actions { + // a drag of a property is one undo step: the first change of each press begins a new one, + // the rest fold into it (like Effect Controls, #201) + if cmd == "graphics.set" && ui.ctx().input(|i| i.pointer.any_down()) { + let key = egui::Id::new("gfx-props-drag-step"); + let press = ui.ctx().input(|i| i.pointer.press_start_time()); + let begun = ui.ctx().data(|d| d.get_temp::>(key)).flatten(); + p["merge"] = json!(true); + p["begin"] = json!(begun != press); + ui.ctx().data_mut(|d| d.insert_temp(key, press)); + } if let Err(e) = app.session.execute(&cmd, p) { app.ui.status = e.to_string(); } diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index 91613447a..bcb386fbb 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -48,7 +48,9 @@ pub mod timeline_captions; pub mod timeline_hit; pub mod timeline_volume; pub mod tools; +pub mod transition_controls; pub mod trim_monitor; +pub mod tts; pub mod voiceover; pub mod workspaces; @@ -98,6 +100,7 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, p: PanelKind, rect: Rect) PanelKind::ReferenceMonitor => reference::show(app, ui, rect), PanelKind::Text => text::show(app, ui, rect), PanelKind::EssentialSound => essential_sound::show(app, ui, rect), + PanelKind::TextToSpeech => tts::show(app, ui, rect), other => crate::dock::placeholder(ui, rect, &app.tokens, &tlf!("{panel} — coming in a later milestone", panel = crate::i18n::t(other.title()))), } } diff --git a/crates/ui-egui/src/panels/timeline.rs b/crates/ui-egui/src/panels/timeline.rs index cfc58e2bb..d59978393 100644 --- a/crates/ui-egui/src/panels/timeline.rs +++ b/crates/ui-egui/src/panels/timeline.rs @@ -130,6 +130,16 @@ pub enum Drag { /// As in [`Drag::Trim`]. from: Tick, }, + /// A transition dragged by an end (`edge`: its duration) or its middle (slides it over the + /// cut); `start` / `duration` are where it is shown now, committed on release (#224). + Transition { + id: filmcraft_project::TransitionId, + edge: Option, + /// As in [`Drag::Trim`]. + from: Tick, + start: Tick, + duration: Tick, + }, Pan { last: Pos2, }, @@ -424,13 +434,22 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { } // selected edit points (trim mode): red brackets for ripple, yellow for roll / regular trim super::trim_monitor::paint_edit_points(app, &p, &seq, tr, r, &layout); - // transitions + // transitions (one being dragged where the drag puts it; the selected ones outlined) for trn in &tr.transitions { - let x0 = layout.x_of(trn.start); - let x1 = layout.x_of(trn.end()); + let (start, end) = match &app.tl.drag { + Some(Drag::Transition { id, start, duration, .. }) if *id == trn.id => (*start, *start + *duration), + _ => (trn.start, trn.end()), + }; + let x0 = layout.x_of(start); + let x1 = layout.x_of(end); let tr_rect = Rect::from_min_max(pos2(x0, r.rect.min.y + 17.0), pos2(x1, r.rect.max.y - 1.0)); - draw_transition(&p, tr_rect, trn, &t); + let selected = app.session.state.transition_selection.contains(&trn.id); + draw_transition(&p, tr_rect, trn, selected, &t); app.auto.add(&format!("timeline.transition.{}", trn.id.0), tr_rect, &trn.effect.effect); + // its ends, for dragging the duration (the same zones as the hit test) + let zone = super::timeline_hit::EDGE_PX.min(tr_rect.width() / 3.0); + app.auto.add(&format!("timeline.transition.{}.in", trn.id.0), Rect::from_min_max(tr_rect.min, pos2(x0 + zone, tr_rect.max.y)), "start"); + app.auto.add(&format!("timeline.transition.{}.out", trn.id.0), Rect::from_min_max(pos2(x1 - zone, tr_rect.min.y), tr_rect.max), "end"); } } @@ -902,7 +921,7 @@ fn paint_hatch(p: &egui::Painter, r: Rect) { } } -fn draw_transition(p: &egui::Painter, r: Rect, trn: &filmcraft_project::Transition, _t: &Tokens) { +fn draw_transition(p: &egui::Painter, r: Rect, trn: &filmcraft_project::Transition, selected: bool, t: &Tokens) { let cp = p.with_clip_rect(r.intersect(p.clip_rect())); cp.rect_filled(r, 0.0, Color32::from_black_alpha(90)); paint_hatch(p, r); @@ -924,6 +943,9 @@ fn draw_transition(p: &egui::Painter, r: Rect, trn: &filmcraft_project::Transiti cp.add(egui::Shape::line(b, Stroke::new(1.0, Color32::WHITE))); } cp.rect_stroke(r, 0.0, Stroke::new(1.0, Color32::from_rgb(0xeb, 0xeb, 0xeb)), StrokeKind::Inside); + if selected { + cp.rect_stroke(r, 0.0, Stroke::new(2.0, t.accent), StrokeKind::Inside); + } if r.width() > 34.0 && !audio { let name = trn.effect.def().map(|d| d.name).unwrap_or(&trn.effect.effect); let tr = Rect::from_min_size(r.min + vec2(2.0, 2.0), vec2((r.width() - 4.0).min(80.0), 13.0)); @@ -1475,7 +1497,7 @@ pub fn hit_json(app: &FilmcraftApp, pos: Pos2, mods: egui::Modifiers) -> Value { Hit::Clip { track, clip, edge } => { json!({"hit": "clip", "track": track.0, "clip": clip.0, "edge": edge.map(|e| format!("{e:?}")), "kind": kind, "time": t.0}) } - Hit::Transition { track, id } => json!({"hit": "transition", "track": track.0, "transition": id.0}), + Hit::Transition { track, id, edge } => json!({"hit": "transition", "track": track.0, "transition": id.0, "edge": edge.map(|e| format!("{e:?}"))}), Hit::Empty { track } => json!({"hit": "empty", "track": track.0, "time": t.0}), Hit::None => json!({"hit": "none"}), } @@ -1826,6 +1848,8 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & let g = grab_at(seq, layout, p, tool, mods, roll_ripple); let cur = match (tool, g) { (_, Grab::Edge { .. }) => CursorIcon::ResizeColumn, + (Tool::Selection, Grab::Other(Hit::Transition { edge: Some(_), .. })) => CursorIcon::ResizeColumn, + (Tool::Selection, Grab::Other(Hit::Transition { edge: None, .. })) => CursorIcon::Grab, (_, Grab::Volume { .. }) if tool == Tool::Pen || mods.command => CursorIcon::Crosshair, (_, Grab::Volume { .. }) => CursorIcon::ResizeVertical, (Tool::Razor, Grab::Other(Hit::Clip { .. })) => CursorIcon::Crosshair, @@ -1943,6 +1967,7 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & } else if !sel.contains(&clip) { if mods.alt { app.session.state.selection = vec![clip]; + app.session.state.transition_selection.clear(); } else { let _ = app.session.execute("timeline.select", json!({"clips": [clip.0]})); } @@ -1954,6 +1979,22 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & None } } + // click a transition to select it (Shift adds / removes); drag an end to change its + // duration or its middle to slide it over the cut (#430, #224) + (Tool::Selection, Grab::Other(Hit::Transition { id, edge, .. })) => { + // a drag of a selected transition (Shift too) moves it rather than toggling it out first + if !(app.session.state.transition_selection.contains(&id) && resp.drag_started()) { + let r = app.session.execute("timeline.select", json!({"transitions": [id.0], "toggle": mods.shift})); + if let Err(e) = r { + app.ui.status = e.to_string(); + } + } + let span = seq.all_tracks().find_map(|tr| tr.transitions.iter().find(|x| x.id == id)).map(|x| (x.start, x.duration)); + match span { + Some((start, duration)) if resp.drag_started() => Some(Drag::Transition { id, edge, from: t, start, duration }), + _ => None, + } + } (_, Grab::Other(Hit::Transition { .. })) => None, (_, Grab::Other(Hit::Empty { .. })) => { if resp.drag_started() { @@ -1961,6 +2002,7 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & } else { app.session.state.selection.clear(); app.session.state.edit_points.clear(); + app.session.state.transition_selection.clear(); None } } @@ -2048,6 +2090,14 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & let origin = ctx.input(|i| i.pointer.press_origin()).map(|o| layout.tick_at(o.x)).unwrap_or(t_here); Some(Drag::Slide { clip, delta: rate.snap_nearest(t_here - origin) }) } + Drag::Transition { id, edge, from, start, duration } => { + let delta = rate.snap_nearest(t_here - from); + let span = seq.all_tracks().find_map(|tr| { + tr.transitions.iter().find(|x| x.id == id).and_then(|x| filmcraft_edit::transitions::drag_span(tr, x, edge, delta, rate.frame_duration())) + }); + let (start, duration) = span.unwrap_or((start, duration)); + Some(Drag::Transition { id, edge, from, start, duration }) + } Drag::Volume(v) => Some(Drag::Volume(super::timeline_volume::drag(app, ui, seq, layout, v, p.y))), Drag::Marquee { start } => { let r = Rect::from_two_pos(start, p); @@ -2095,6 +2145,10 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & Drag::Roll { left, right, delta, .. } if delta != Tick::ZERO => Some(app.session.execute("timeline.roll", json!({"left": left.0, "right": right.0, "delta": delta.0}))), Drag::Slip { clip, delta } if delta != Tick::ZERO => Some(app.session.execute("timeline.slip", json!({"clip": clip.0, "delta": delta.0}))), Drag::Slide { clip, delta } if delta != Tick::ZERO => Some(app.session.execute("timeline.slide", json!({"clip": clip.0, "delta": delta.0}))), + Drag::Transition { id, start, duration, .. } => { + let moved = seq.all_tracks().find_map(|tr| tr.transitions.iter().find(|x| x.id == id)).is_some_and(|x| (x.start, x.duration) != (start, duration)); + moved.then(|| app.session.execute("sequence.setTransition", json!({"transition": id.0, "start": start.0, "duration": duration.0}))) + } Drag::Marquee { start } => { if let Some(end) = resp.interact_pointer_pos() { let r = Rect::from_two_pos(start, end); @@ -2118,6 +2172,15 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & } } + // ---- double-click a transition: Set Transition Duration (Premiere) + if resp.double_clicked() + && tool == Tool::Selection + && let Some(p) = resp.interact_pointer_pos() + && let Hit::Transition { id, .. } = hit(seq, layout, p) + { + crate::panels::transition_controls::open_duration(app, id); + } + // ---- double-click a nested sequence clip: open it in its own Timeline tab if resp.double_clicked() && tool == Tool::Selection @@ -2342,9 +2405,65 @@ fn interact(app: &mut FilmcraftApp, ui: &mut egui::Ui, seq: &Sequence, layout: & } } } + // ---- OS file drops imported with "drop imports to the timeline": placed at the drop point, one after another + if let Some((items, p)) = app.pending_timeline_drop.clone() + && layout.content.contains(p) + { + app.pending_timeline_drop = None; + let row = layout.row_at(p.y).cloned().or_else(|| layout.rows.iter().find(|r| r.kind == TrackKind::Video).cloned()); + if let Some(row) = row { + let (vt, at) = match row.kind { + TrackKind::Video => (Some(row.track.0), seq.audio_tracks.get(row.index).or(seq.audio_tracks.first()).map(|t| t.id.0)), + TrackKind::Audio => (seq.video_tracks.get(row.index).or(seq.video_tracks.first()).map(|t| t.id.0), Some(row.track.0)), + }; + let still = app.session.prefs.timeline.still_duration(rate); + let durations: Vec = items + .iter() + .map(|&item| { + let is_still = app.session.project.item(item).and_then(|i| i.as_media()).is_some_and(|m| m.info.kind == filmcraft_media::MediaKind::Still); + app.session.project.item(item).map(|i| i.duration()).filter(|d| d.0 > 0 && !is_still).unwrap_or(still) + }) + .collect(); + let start = snap(app, seq, layout, rate.snap_nearest(layout.tick_at(p.x).max(Tick::ZERO)), &[]); + for (item, t) in items.iter().zip(chain_starts(start, &durations)) { + let params = json!({"item": item.0, "track": vt, "audioTrack": at, "time": t.0, "insert": false}); + if let Err(e) = app.session.execute("timeline.place", params) { + app.ui.status = e.to_string(); + } + } + } + } + // preview of OS files being dragged over the Timeline: a placeholder where the drop would start + if let Some((count, p)) = app.file_drag_hover + && layout.content.contains(p) + { + let row = layout.row_at(p.y).cloned().or_else(|| layout.rows.iter().find(|r| r.kind == TrackKind::Video).cloned()); + if let Some(row) = row { + let t = snap(app, seq, layout, rate.snap_nearest(layout.tick_at(p.x).max(Tick::ZERO)), &[]); + let x = layout.x_of(t); + let r = Rect::from_min_max(pos2(x, row.rect.min.y + 1.0), pos2((x + 120.0).min(layout.content.max.x), row.rect.max.y - 1.0)); + ui.painter().rect_filled(r, 3.0, Color32::from_white_alpha(40)); + ui.painter().rect_stroke(r, 3.0, Stroke::new(1.5, Color32::WHITE), StrokeKind::Inside); + let label = if count == 1 { tl!("1 file").to_string() } else { tlf!("{n} files", n = count) }; + ui.painter().text(r.left_top() + vec2(4.0, 2.0), Align2::LEFT_TOP, label, Tokens::ui(10.0), Color32::WHITE); + } + } app.auto.add("timeline.tracks", layout.content, "tracks"); } +/// Start tick of each clip when the clips are laid end to end from `start`. Saturates rather than overflows. +pub(crate) fn chain_starts(start: Tick, durations: &[Tick]) -> Vec { + let mut t = start; + durations + .iter() + .map(|d| { + let at = t; + t = Tick(t.0.saturating_add(d.0.max(0))); + at + }) + .collect() +} + #[cfg(test)] mod waveform_tests { use super::waveform_display_gain; @@ -2367,3 +2486,41 @@ mod waveform_tests { assert!((db(waveform_display_gain(1.0, -6.0)) - (-6.0)).abs() < 0.1); } } + +#[cfg(test)] +mod chain_starts_tests { + use super::chain_starts; + use filmcraft_time::Tick; + + #[test] + fn empty_list_has_no_starts() { + assert!(chain_starts(Tick(100), &[]).is_empty()); + } + + #[test] + fn one_clip_starts_at_the_drop_point() { + assert_eq!(chain_starts(Tick(7), &[Tick(30)]), vec![Tick(7)]); + } + + #[test] + fn several_clips_are_laid_end_to_end() { + let durations = [Tick(10), Tick(25), Tick(5)]; + assert_eq!(chain_starts(Tick(100), &durations), vec![Tick(100), Tick(110), Tick(135)]); + } + + #[test] + fn start_near_i64_max_saturates_without_panicking() { + let starts = chain_starts(Tick(i64::MAX - 5), &[Tick(10), Tick(10), Tick(10)]); + assert_eq!(starts, vec![Tick(i64::MAX - 5), Tick(i64::MAX), Tick(i64::MAX)]); + } + + #[test] + fn zero_length_clips_share_one_start() { + assert_eq!(chain_starts(Tick(42), &[Tick(0), Tick(0), Tick(9)]), vec![Tick(42), Tick(42), Tick(42)]); + } + + #[test] + fn negative_durations_count_as_zero() { + assert_eq!(chain_starts(Tick(5), &[Tick(-3), Tick(4)]), vec![Tick(5), Tick(5)]); + } +} diff --git a/crates/ui-egui/src/panels/timeline_hit.rs b/crates/ui-egui/src/panels/timeline_hit.rs index 1716649fe..16dd5f712 100644 --- a/crates/ui-egui/src/panels/timeline_hit.rs +++ b/crates/ui-egui/src/panels/timeline_hit.rs @@ -50,9 +50,21 @@ pub(crate) fn edge_geometry(seq: &Sequence, layout: &Layout, track: TrackId, cli #[derive(Clone, Copy, Debug, PartialEq)] pub enum Hit { Ruler, - Clip { track: TrackId, clip: ClipId, edge: Option }, - Transition { track: TrackId, id: TransitionId }, - Empty { track: TrackId }, + Clip { + track: TrackId, + clip: ClipId, + edge: Option, + }, + /// A transition; `edge` is its start (`In`) or end (`Out`) when the press is on one of its + /// ends (dragging it changes the duration), `None` on its middle (dragging slides it). + Transition { + track: TrackId, + id: TransitionId, + edge: Option, + }, + Empty { + track: TrackId, + }, None, } @@ -69,7 +81,16 @@ pub fn hit(seq: &Sequence, layout: &Layout, pos: Pos2) -> Hit { let x0 = layout.x_of(trn.start); let x1 = layout.x_of(trn.end()); if pos.x >= x0 && pos.x <= x1 && pos.y > row.rect.min.y + 17.0 { - return Hit::Transition { track: row.track, id: trn.id }; + // the ends are edge zones, narrower on a short transition so its middle stays grabbable + let zone = EDGE_PX.min((x1 - x0) / 3.0); + let edge = if pos.x - x0 <= zone { + Some(Edge::In) + } else if x1 - pos.x <= zone { + Some(Edge::Out) + } else { + None + }; + return Hit::Transition { track: row.track, id: trn.id, edge }; } } if let Some((clip, edge)) = edge_at(tr, layout, pos.x) { diff --git a/crates/ui-egui/src/panels/transition_controls.rs b/crates/ui-egui/src/panels/transition_controls.rs new file mode 100644 index 000000000..80a8a0d4a --- /dev/null +++ b/crates/ui-egui/src/panels/transition_controls.rs @@ -0,0 +1,305 @@ +//! A selected transition (#430): its Effect Controls (Duration, Alignment, Reverse and its own +//! settings, as Premiere's Change transition settings / Align transitions / Change transition +//! duration pages describe them) and the Set Transition Duration dialog a double-click on it in +//! the Timeline opens. Every change is one `sequence.setTransition`; dragging a value is one undo +//! step. +//! +//! Not here yet: the A/B preview thumbnails with their Start / End sliders, Show Actual Sources, +//! and dragging the transition in Effect Controls' own time ruler (the Timeline does that). +//! +//! Automation ids: `effectControls.transition.duration`, `.alignment` (+ `.alignment.option.` while open), `.reverse`, `.reset`, `.param.` (a point's are `.param..x` / +//! `.y`; a list's entries `.param..option.` while open); the dialog's +//! `transitionDuration.value`, `.ok`, `.cancel`. + +use egui::{Rect, RichText, vec2}; +use filmcraft_edit::transitions::{self as trn, Alignment}; +use filmcraft_project::{ParamKind, ParamValue, Transition, TransitionId}; +use filmcraft_time::FrameRate; +use serde_json::{Value, json}; + +use crate::FilmcraftApp; +use crate::state::TransitionDurationDraft; + +type Elems = Vec<(String, Rect, String)>; + +/// The selected transition, when it still exists, with its alignment and how many frames it may +/// last at most (the clips it joins). +fn lookup(app: &FilmcraftApp, id: TransitionId) -> Option<(Transition, Alignment, i64)> { + let seq = app.session.active_sequence()?; + let rate = seq.settings.frame_rate.sane(); + seq.all_tracks().find_map(|tr| { + let x = tr.transitions.iter().find(|x| x.id == id)?; + let (lo, cut, hi) = trn::bounds(tr, x)?; + let max = rate.frame_at(hi - lo).max(1); + Some((x.clone(), trn::alignment(x, cut, rate.frame_duration()), max)) + }) +} + +/// The first selected transition that still exists. +pub fn selected(app: &FilmcraftApp) -> Option { + app.session.state.transition_selection.iter().copied().find(|id| lookup(app, *id).is_some()) +} + +fn timecode(frames: f64, rate: FrameRate, drop_frame: bool) -> String { + filmcraft_time::format_timecode_frames(frames.round() as i64, rate, drop_frame) +} + +/// A duration field in frames that reads and takes timecode (or a plain frame count). +fn duration_field(ui: &mut egui::Ui, frames: &mut i64, max: i64, rate: FrameRate, drop_frame: bool) -> egui::Response { + ui.add(egui::DragValue::new(frames).range(1..=max.max(1)).speed(0.25).custom_formatter(move |v, _| timecode(v, rate, drop_frame)).custom_parser(move |s| { + match s.trim().parse::() { + Ok(n) => Some(n as f64), + Err(_) => filmcraft_time::parse_timecode(s, rate, drop_frame, 0).ok().map(|n| n as f64), + } + })) +} + +/// Effect Controls for a selected transition. `rect` is the panel. +pub fn effect_controls(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect, id: TransitionId) { + let t = app.tokens; + let Some((x, align, max_frames)) = lookup(app, id) else { + crate::dock::placeholder(ui, rect, &t, tl!("(no clip selected)")); + return; + }; + let Some(seq) = app.session.active_sequence() else { return }; + let (rate, drop_frame) = (seq.settings.frame_rate.sane(), seq.settings.drop_frame); + let (fw, fh) = (f64::from(seq.settings.width), f64::from(seq.settings.height)); + let seq_name = app.session.state.active_sequence.and_then(|i| app.session.project.item(i)).map(|i| i.name.clone()).unwrap_or_default(); + let def = x.effect.def(); + let name = def.map_or(x.effect.effect.as_str(), |d| d.name).to_string(); + let one_sided = x.from.is_none() || x.to.is_none(); + let mut changes: Vec = Vec::new(); + let mut elems: Elems = Vec::new(); + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(rect.shrink2(vec2(10.0, 6.0))).id_salt(("tr-ec", id.0))); + child.label(RichText::new(format!("{seq_name} · {name}")).color(t.text_dim)); + child.add_space(4.0); + egui::ScrollArea::vertical().id_salt(("tr-ec-scroll", id.0)).auto_shrink([false, false]).show(&mut child, |ui| { + egui::Grid::new(("tr-ec-grid", id.0)).num_columns(2).spacing([14.0, 8.0]).show(ui, |ui| { + ui.label(tl!("Duration")); + let mut frames = rate.frame_at(x.duration).max(1); + let r = duration_field(ui, &mut frames, max_frames, rate, drop_frame); + elems.push(("effectControls.transition.duration".into(), r.rect, timecode(frames as f64, rate, drop_frame))); + if r.changed() { + changes.push(json!({"frames": frames, "merge": true, "begin": r.drag_started() || !r.dragged()})); + } + ui.end_row(); + + ui.label(tl!("Alignment")); + let shown = align.label(); + let r = ui + .add_enabled_ui(!one_sided, |ui| { + egui::ComboBox::from_id_salt(("tr-align", id.0)) + .selected_text(shown) + .width(170.0) + .show_ui(ui, |ui| { + for a in Alignment::MENU { + let o = ui.selectable_label(a == align, a.label()); + if o.clicked() && a != align { + changes.push(json!({"align": a.id()})); + } + elems.push((format!("effectControls.transition.alignment.option.{}", a.id()), o.rect, a.label().to_string())); + } + }) + .response + }) + .inner; + elems.push(("effectControls.transition.alignment".into(), r.rect, shown.to_string())); + ui.end_row(); + + ui.label(tl!("Reverse")); + let mut rev = x.reverse; + let r = ui.checkbox(&mut rev, ""); + elems.push(("effectControls.transition.reverse".into(), r.rect, rev.to_string())); + if r.changed() { + changes.push(json!({"reverse": rev})); + } + ui.end_row(); + + for pd in def.map(|d| d.params.as_slice()).unwrap_or(&[]) { + let value = x.effect.params.get(pd.id).map_or(&pd.default, |p| &p.value); + ui.label(crate::i18n::t(pd.label)); + if let Some(v) = param_widget(ui, &mut elems, pd.id, &pd.kind, value, (fw, fh)) { + changes.push(v); + } + ui.end_row(); + } + }); + ui.add_space(8.0); + let r = ui.button(tl!("Reset")); + elems.push(("effectControls.transition.reset".into(), r.rect, "Reset".into())); + if r.clicked() { + changes.push(json!({"reset": true})); + } + }); + for (eid, r, l) in elems { + app.auto.add(&eid, r, &l); + } + for mut c in changes { + c["transition"] = json!(id.0); + if let Err(e) = app.session.execute("sequence.setTransition", c) { + app.ui.status = e.to_string(); + } + } +} + +/// One of a transition's own settings. Returns the change, as `sequence.setTransition` params. +fn param_widget(ui: &mut egui::Ui, elems: &mut Elems, pid: &str, kind: &ParamKind, value: &ParamValue, frame: (f64, f64)) -> Option { + let eid = format!("effectControls.transition.param.{pid}"); + let drag = |r: &egui::Response| json!({"merge": true, "begin": r.drag_started() || !r.dragged()}); + match (kind, value) { + (ParamKind::Float { min, max, soft_min, soft_max, unit, decimals }, ParamValue::Float(v)) => { + let mut v = *v; + let speed = ((soft_max - soft_min) / 200.0).max(0.001); + let r = ui.add(egui::DragValue::new(&mut v).range(*min..=*max).speed(speed).max_decimals(usize::from(*decimals)).suffix(*unit)); + elems.push((eid, r.rect, format!("{v}"))); + r.changed().then(|| { + let mut c = drag(&r); + c["params"] = json!({ pid: v }); + c + }) + } + (ParamKind::Angle, ParamValue::Float(v)) => { + let mut v = *v; + let r = ui.add(egui::DragValue::new(&mut v).speed(0.5).suffix("°")); + elems.push((eid, r.rect, format!("{v}"))); + r.changed().then(|| { + let mut c = drag(&r); + c["params"] = json!({ pid: v }); + c + }) + } + (ParamKind::Bool, ParamValue::Bool(b)) => { + let mut b = *b; + let r = ui.checkbox(&mut b, ""); + elems.push((eid, r.rect, b.to_string())); + r.changed().then(|| json!({"params": { pid: b }})) + } + (ParamKind::Choice(opts), ParamValue::Choice(c)) => { + let shown = opts.get(*c as usize).copied().unwrap_or(""); + let mut picked = None; + let mut options = Vec::new(); + let r = egui::ComboBox::from_id_salt(("tr-param", pid.to_string())).selected_text(crate::i18n::t(shown)).show_ui(ui, |ui| { + for (i, o) in opts.iter().enumerate() { + let r = ui.selectable_label(i == *c as usize, crate::i18n::t(o)); + if r.clicked() && i != *c as usize { + picked = Some(i); + } + options.push((format!("{eid}.option.{i}"), r.rect, o.to_string())); + } + }); + elems.extend(options); + elems.push((eid, r.response.rect, shown.to_string())); + picked.map(|i| json!({"params": { pid: i }})) + } + (ParamKind::Color, ParamValue::Color(c)) => { + let mut c = *c; + let r = ui.color_edit_button_rgba_unmultiplied(&mut c); + elems.push((eid, r.rect, format!("{c:?}"))); + r.changed().then(|| { + let mut v = drag(&r); + v["params"] = json!({ pid: [c[0], c[1], c[2], c[3]] }); + v + }) + } + (ParamKind::Point, ParamValue::Vec2(p)) => { + // an automatic point (NaN) is the frame's centre until it is moved + let (mut px, mut py) = (if p.x.is_nan() { frame.0 / 2.0 } else { p.x }, if p.y.is_nan() { frame.1 / 2.0 } else { p.y }); + let mut out = None; + ui.horizontal(|ui| { + let rx = ui.add(egui::DragValue::new(&mut px).speed(1.0).prefix("x ")); + let ry = ui.add(egui::DragValue::new(&mut py).speed(1.0).prefix("y ")); + elems.push((format!("{eid}.x"), rx.rect, format!("{px}"))); + elems.push((format!("{eid}.y"), ry.rect, format!("{py}"))); + for r in [&rx, &ry] { + if r.changed() { + let mut v = drag(r); + v["params"] = json!({ pid: [px, py] }); + out = Some(v); + } + } + }); + out + } + (_, v) => { + let r = ui.label(RichText::new(format!("{v:?}")).weak()); + elems.push((eid, r.rect, String::new())); + None + } + } +} + +/// Double-click a transition in the Timeline: Set Transition Duration. +pub fn open_duration(app: &mut FilmcraftApp, id: TransitionId) { + let Some((x, ..)) = lookup(app, id) else { return }; + let rate = app.session.active_sequence().map(|q| q.settings.frame_rate.sane()).unwrap_or_default(); + app.ui.transition_duration = TransitionDurationDraft { transition: id.0, frames: rate.frame_at(x.duration).max(1) }; + let _ = app.session.execute("timeline.select", json!({"transitions": [id.0]})); + app.dialog = Some(crate::Dialog::TransitionDuration); +} + +/// The Set Transition Duration dialog. Returns whether it stays open. +pub fn duration_dialog(app: &mut FilmcraftApp, ctx: &egui::Context) -> bool { + let mut d = app.ui.transition_duration.clone(); + let id = TransitionId(d.transition); + let Some((_, _, max_frames)) = lookup(app, id) else { return false }; + let Some(seq) = app.session.active_sequence() else { return false }; + let (rate, drop_frame) = (seq.settings.frame_rate.sane(), seq.settings.drop_frame); + let accent = app.tokens.accent; + let (mut keep, mut apply) = (true, false); + let mut elems: Elems = Vec::new(); + egui::Window::new(tl!("Set Transition Duration")) + .collapsible(false) + .resizable(false) + .default_width(260.0) + .anchor(egui::Align2::CENTER_CENTER, [0.0, 0.0]) + .show(ctx, |ui| { + ui.horizontal(|ui| { + ui.label(tl!("Duration:")); + let r = duration_field(ui, &mut d.frames, max_frames, rate, drop_frame); + elems.push(("transitionDuration.value".into(), r.rect, timecode(d.frames as f64, rate, drop_frame))); + }); + ui.add_space(8.0); + // one row high: a right-to-left layout would otherwise take the window's whole height + let row = egui::vec2(ui.available_width().max(220.0), ui.spacing().interact_size.y); + ui.allocate_ui_with_layout(row, egui::Layout::right_to_left(egui::Align::Center), |ui| { + let o = ui.add(egui::Button::new(RichText::new(tl!("OK")).color(egui::Color32::WHITE)).fill(accent)); + elems.push(("transitionDuration.ok".into(), o.rect, "OK".into())); + apply |= o.clicked(); + let c = ui.button(tl!("Cancel")); + elems.push(("transitionDuration.cancel".into(), c.rect, "Cancel".into())); + keep &= !c.clicked(); + }); + }); + for (eid, r, l) in elems { + app.auto.add(&eid, r, &l); + } + if ctx.input(|i| i.key_pressed(egui::Key::Escape)) { + keep = false; + } + if ctx.input(|i| i.key_pressed(egui::Key::Enter)) && !ctx.egui_wants_keyboard_input() { + apply = true; + } + if apply { + let cur = lookup(app, id).map(|(x, ..)| rate.frame_at(x.duration)); + if cur != Some(d.frames) + && let Err(e) = app.session.execute("sequence.setTransition", json!({"transition": id.0, "frames": d.frames})) + { + app.ui.status = e.to_string(); + } + keep = false; + } + app.ui.transition_duration = d; + keep +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn durations_read_as_timecode() { + assert_eq!(timecode(24.0, FrameRate::FPS_24, false), "00:00:01:00"); + assert_eq!(timecode(30.0, FrameRate::FPS_29_97, true), "00;00;01;00"); + } +} diff --git a/crates/ui-egui/src/panels/tts.rs b/crates/ui-egui/src/panels/tts.rs new file mode 100644 index 000000000..8bc12e7cb --- /dev/null +++ b/crates/ui-egui/src/panels/tts.rs @@ -0,0 +1,465 @@ +//! Text to Speech panel: write what the narrator says, pick a language, voice, vocal pitch and pace, +//! hear it, and save it as a narration clip. Selecting a narration clip on the timeline loads its +//! script and settings here (and brings the panel forward), and Save regenerates it in place. +//! Every action is a `tts.*` command, so the panel, MCP and the control channel share one path. +//! +//! Synthesis runs as a background job (`tts.render`); when it finishes, the panel runs the action +//! (preview, create or edit), which then uses the cached audio, so the window never stalls. The +//! natural voices need a one-time download that the user confirms in the panel (size, source, +//! licence); it runs as a job with progress and Cancel. +//! +//! Automation ids: `tts.language`, `tts.voice`, `tts.hearVoice`, `tts.advanced` (twirl), +//! `tts.pitch`, `tts.pace`, `tts.text`, `tts.addPause`, `tts.preview`, `tts.save`, `tts.new`, +//! `tts.download`, `tts.download.confirm`, `tts.download.cancel`. + +use egui::{Align2, Rect, Sense, Stroke, StrokeKind, pos2, vec2}; +use filmcraft_project::{ClipId, VocalPitch}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +use crate::FilmcraftApp; +use crate::icons::{self, Icon}; +use crate::theme::Tokens; + +const PAD: f32 = 14.0; +const FOOTER_H: f32 = 48.0; + +/// What the panel is writing (UI state, serde so agents can read it). +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct TtsDraft { + pub text: String, + pub language: String, + pub voice: String, + pub pitch: VocalPitch, + pub pace: f64, + pub advanced_open: bool, + /// The narration clip and item the draft was loaded from (None: a new narration). + pub loaded_from: Option<(u64, u64)>, + /// An action waiting for its synthesis job. + pub pending: Option, + /// The download confirmation is showing. + pub confirm_download: bool, + /// The running natural-voice download job. + pub download_job: Option, +} + +/// A command to run when the synthesis job `job` finishes. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Pending { + pub cmd: String, + pub params: serde_json::Value, + /// Play the preview afterwards. + pub play: bool, + pub job: u64, +} + +impl Default for TtsDraft { + fn default() -> Self { + TtsDraft { + text: String::new(), + language: "en-US".into(), + voice: if natural_installed() { "kokoro-heart".into() } else { filmcraft_tts::default_voice_id().into() }, + pitch: VocalPitch::Default, + pace: 1.0, + advanced_open: true, + loaded_from: None, + pending: None, + confirm_download: false, + download_job: None, + } + } +} + +impl TtsDraft { + fn params(&self) -> serde_json::Value { + json!({"text": self.text, "voice": self.voice, "pitch": self.pitch.id(), "pace": self.pace}) + } +} + +/// The natural voice package is downloaded. +fn natural_installed() -> bool { + filmcraft_engine::transcript::models_dir().as_deref().is_some_and(filmcraft_tts::catalog::installed) +} + +/// State of job `id`: `None` while running, else its result. +fn job_outcome(app: &FilmcraftApp, id: u64) -> Option> { + use std::sync::atomic::Ordering; + let Some(j) = app.session.jobs.iter().find(|j| j.id == id) else { return Some(Err("the job disappeared".into())) }; + if !j.progress.finished.load(Ordering::Relaxed) { + return None; + } + Some(match &*j.result.lock().unwrap_or_else(std::sync::PoisonError::into_inner) { + Some(Ok(_)) => Ok(()), + Some(Err(e)) => Err(e.clone()), + None => Err("the job stopped".into()), + }) +} + +/// Start an action: synthesize in the background (or reuse the cache), then run `cmd`. +fn start(app: &mut FilmcraftApp, cmd: &str, params: serde_json::Value, play: bool) { + let mut render = params.clone(); + if let Some(o) = render.as_object_mut() { + o.remove("dir"); + o.remove("time"); + } + match app.session.execute("tts.render", render) { + Ok(r) => match r["job"].as_u64() { + Some(job) => { + app.ui.tts.pending = Some(Pending { cmd: cmd.to_string(), params, play, job }); + app.ui.status = "Synthesizing narration…".into(); + } + None => run_now(app, cmd, params, play), + }, + Err(e) => app.ui.status = e.to_string(), + } +} + +fn run_now(app: &mut FilmcraftApp, cmd: &str, params: serde_json::Value, play: bool) { + match app.session.execute(cmd, params) { + Ok(r) => { + if play { + if let Err(e) = app.play_tts_preview() { + app.ui.status = e; + } + } else { + app.ui.status = match cmd { + "tts.create" => format!("Narration added on {}", r["track"].as_str().unwrap_or("an audio track")), + _ => "Narration updated".into(), + }; + } + } + Err(e) => app.ui.status = e.to_string(), + } +} + +/// Finish a pending action whose synthesis job is done. +fn poll(app: &mut FilmcraftApp) { + if let Some(p) = app.ui.tts.pending.clone() { + match job_outcome(app, p.job) { + None => {} + Some(Ok(())) => { + app.ui.tts.pending = None; + run_now(app, &p.cmd, p.params, p.play); + } + Some(Err(e)) => { + app.ui.tts.pending = None; + app.ui.status = format!("Text to Speech: {e}"); + } + } + } + if let Some(j) = app.ui.tts.download_job + && let Some(r) = job_outcome(app, j) + { + app.ui.tts.download_job = None; + app.ui.status = match r { + Ok(()) => "Natural voices installed".into(), + Err(e) => format!("Download of the natural voices failed: {e}"), + }; + } +} + +/// The selected narration clip and its item, if any. +fn selected_narration(app: &FilmcraftApp) -> Option<(ClipId, u64)> { + let clip = filmcraft_engine::narration::target_clip(&app.session, &serde_json::Value::Null)?; + let item = app.session.active_sequence()?.find_item(clip)?.1.item; + Some((clip, item.0)) +} + +/// Load the selected narration clip into the draft when the selection moves to a different one, +/// and bring the panel forward. Runs every frame, also when the panel is hidden. +pub fn follow_selection(app: &mut FilmcraftApp) { + poll(app); + let sel = selected_narration(app); + match sel { + Some((clip, item)) if app.ui.tts.loaded_from != Some((clip.0, item)) => { + let first_time = app.ui.tts.loaded_from.map(|(c, _)| c) != Some(clip.0); + if let Some(n) = app.session.project.narrations.get(&filmcraft_project::ItemId(item)).cloned() { + let d = &mut app.ui.tts; + d.text = n.text; + d.language = n.language; + d.voice = n.voice; + d.pitch = n.pitch; + d.pace = n.pace; + d.loaded_from = Some((clip.0, item)); + if first_time { + app.show_panel(crate::dock::PanelKind::TextToSpeech); + } + } + } + None if app.ui.tts.loaded_from.is_some() => { + // deselected: keep the text so it can be saved as a new narration + app.ui.tts.loaded_from = None; + } + _ => {} + } +} + +enum Action { + Run(&'static str, serde_json::Value, bool), + New, + Download, + CancelDownload, +} + +pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) { + let t = app.tokens; + let painter = ui.painter().clone(); + painter.rect_filled(rect, 0.0, t.panel_bg); + let models = filmcraft_engine::transcript::models_dir(); + let voices = filmcraft_tts::voices_in(models.as_deref()); + let installed = models.as_deref().is_some_and(filmcraft_tts::catalog::installed); + let busy = app.ui.tts.pending.is_some(); + let download = app + .ui + .tts + .download_job + .and_then(|id| app.session.jobs.iter().find(|j| j.id == id)) + .map(|j| (j.progress.fraction(), j.progress.status.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone())); + let editing = app.ui.tts.loaded_from.map(|(c, _)| c); + let mut actions: Vec = Vec::new(); + let auto = &mut app.auto; + let d = &mut app.ui.tts; + + let body = Rect::from_min_max(pos2(rect.min.x + PAD, rect.min.y + 8.0), pos2(rect.max.x - PAD, rect.max.y - FOOTER_H)); + let mut child = ui.new_child(egui::UiBuilder::new().max_rect(body).id_salt("tts-body")); + egui::ScrollArea::vertical().auto_shrink([false, false]).show(&mut child, |ui| { + ui.set_max_width(ui.available_width() - 6.0); + let w = ui.available_width(); + // mode line: editing a clip, or writing a new narration + ui.horizontal(|ui| { + let mode = if editing.is_some() { "Editing the selected narration" } else { "New narration" }; + ui.label(egui::RichText::new(mode).color(t.text_dim).size(11.5)); + if editing.is_some() { + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + let b = ui.small_button("New"); + auto.add("tts.new", visible(ui, b.rect), "New narration"); + if b.clicked() { + actions.push(Action::New); + } + }); + } + }); + ui.add_space(6.0); + label(ui, &t, "Language"); + let lang_name = filmcraft_tts::LANGUAGES.iter().find(|(id, _)| *id == d.language).map_or(d.language.as_str(), |(_, n)| n); + let cb = egui::ComboBox::from_id_salt("tts-language").selected_text(lang_name).width(w).show_ui(ui, |ui| { + for (id, name) in filmcraft_tts::LANGUAGES { + if ui.selectable_label(d.language == *id, *name).clicked() { + d.language = id.to_string(); + } + } + }); + auto.add("tts.language", visible(ui, cb.response.rect), "Language"); + ui.add_space(10.0); + label(ui, &t, "Voice"); + let voice_name = voices.iter().find(|v| v.id == d.voice).map_or(d.voice.as_str(), |v| v.name); + let cb = egui::ComboBox::from_id_salt("tts-voice").selected_text(voice_name).width(w).show_ui(ui, |ui| { + for (engine, heading) in [("neural", "Natural voices"), ("basic", "Basic voices (built in)")] { + ui.label(egui::RichText::new(heading).color(t.text_faint).size(11.0)); + for v in voices.iter().filter(|v| v.language == d.language && v.engine == engine) { + let label = if v.engine == "neural" && !v.installed { format!("{} (download)", v.name) } else { v.name.to_string() }; + let r = ui.selectable_label(d.voice == v.id, label).on_hover_text(v.description); + if r.clicked() { + d.voice = v.id.to_string(); + } + } + } + }); + auto.add("tts.voice", visible(ui, cb.response.rect), "Voice"); + let natural = filmcraft_tts::catalog::find(&d.voice).is_some(); + if natural && !installed { + ui.add_space(6.0); + egui::Frame::new().fill(t.field_bg).stroke(egui::Stroke::new(1.0, t.field_border)).corner_radius(4.0).inner_margin(8.0).show(ui, |ui| { + ui.set_width(w - 18.0); + match &download { + Some((frac, status)) => { + ui.label(egui::RichText::new(status).color(t.text).size(11.5)); + ui.add(egui::ProgressBar::new(*frac).show_percentage()); + let b = ui.small_button("Cancel"); + auto.add("tts.download.cancel", visible(ui, b.rect), "Cancel download"); + if b.clicked() { + actions.push(Action::CancelDownload); + } + } + None if d.confirm_download => { + let mb = filmcraft_tts::catalog::size() as f64 / 1e6; + ui.label(egui::RichText::new(format!("Download the natural voices? {mb:.0} MB, once.")).color(t.text).size(12.0).strong()); + ui.label( + egui::RichText::new(format!( + "From {} and github.com/cmusphinx/cmudict (pronunciations). Licence: {}. Every file is checked against a pinned SHA-256.", + filmcraft_tts::catalog::SOURCE, + filmcraft_tts::catalog::LICENSE + )) + .color(t.text_dim) + .size(11.0), + ); + ui.horizontal(|ui| { + let ok = ui.button("Download"); + auto.add("tts.download.confirm", visible(ui, ok.rect), "Download"); + let no = ui.button("Cancel"); + if ok.clicked() { + actions.push(Action::Download); + } + if no.clicked() { + d.confirm_download = false; + } + }); + } + None => { + ui.label(egui::RichText::new("Natural voices need a one-time download.").color(t.text).size(11.5)); + let b = ui.button(format!("Download natural voices ({:.0} MB)…", filmcraft_tts::catalog::size() as f64 / 1e6)); + auto.add("tts.download", visible(ui, b.rect), "Download natural voices"); + if b.clicked() { + d.confirm_download = true; + } + } + } + }); + } + let voice_ready = !natural || installed; + ui.add_space(4.0); + let (r, resp) = ui.allocate_exact_size(vec2(130.0, 22.0), Sense::click()); + let col = if resp.hovered() { t.accent_hover } else { t.accent }; + icons::paint(ui.painter(), Rect::from_center_size(pos2(r.min.x + 8.0, r.center().y), vec2(14.0, 14.0)), Icon::Play, col); + ui.painter().text(pos2(r.min.x + 22.0, r.center().y), Align2::LEFT_CENTER, "Hear this voice", Tokens::semibold(12.0), col); + auto.add("tts.hearVoice", visible(ui, r), "Hear this voice"); + if resp.clicked() && voice_ready && !busy { + actions.push(Action::Run("tts.preview", json!({"sample": true, "voice": d.voice, "pitch": d.pitch.id(), "pace": d.pace}), true)); + } + ui.add_space(8.0); + let (hr, open) = crate::widgets::section_header(ui, egui::Id::new("tts-advanced"), "Advanced", d.advanced_open, &t, true); + auto.add("tts.advanced", visible(ui, hr.rect), "Advanced"); + d.advanced_open = open; + if open { + ui.add_space(4.0); + label(ui, &t, "Vocal pitch"); + let cb = egui::ComboBox::from_id_salt("tts-pitch").selected_text(d.pitch.label()).width(w).show_ui(ui, |ui| { + for p in VocalPitch::ALL { + if ui.selectable_label(d.pitch == p, p.label()).clicked() { + d.pitch = p; + } + } + }); + auto.add("tts.pitch", visible(ui, cb.response.rect), "Vocal pitch"); + ui.add_space(10.0); + label(ui, &t, "Pace"); + ui.spacing_mut().slider_width = w - 56.0; + let s = ui.add(egui::Slider::new(&mut d.pace, filmcraft_tts::MIN_PACE..=filmcraft_tts::MAX_PACE).step_by(0.05).fixed_decimals(2).suffix("×")); + auto.add("tts.pace", visible(ui, s.rect), "Pace"); + let (r, _) = ui.allocate_exact_size(vec2(w - 56.0, 14.0), Sense::hover()); + for (x, l) in [(0.0, "0.5×"), (1.0 / 3.0, "1×"), (1.0, "2×")] { + let px = r.min.x + 6.0 + (r.width() - 12.0) * x; + ui.painter().text(pos2(px, r.center().y), Align2::CENTER_CENTER, l, Tokens::ui(10.5), t.text_faint); + } + } + ui.add_space(12.0); + ui.horizontal(|ui| { + ui.label(egui::RichText::new("Text").color(t.text).size(12.5).strong()); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + let (ir, i) = ui.allocate_exact_size(vec2(14.0, 14.0), Sense::hover()); + icons::paint(ui.painter(), ir, Icon::Info, t.text_faint); + i.on_hover_text("Add pause inserts [pause 1s]. Change the number for a longer or shorter pause, e.g. [pause 500ms] or [pause 2s]."); + let b = ui.small_button("Add pause"); + auto.add("tts.addPause", visible(ui, b.rect), "Add pause"); + if b.clicked() { + if !d.text.is_empty() && !d.text.ends_with(' ') { + d.text.push(' '); + } + d.text.push_str(filmcraft_tts::script::PAUSE_MARKER); + d.text.push(' '); + } + }); + }); + ui.add_space(4.0); + let te = egui::TextEdit::multiline(&mut d.text) + .hint_text("Type what you want the voice to say") + .desired_width(w) + .desired_rows(8) + .char_limit(filmcraft_tts::MAX_TEXT_BYTES); + let r = ui.add(te); + auto.add("tts.text", visible(ui, r.rect), "Text"); + }); + + // footer: Preview | Save + let fy = rect.max.y - FOOTER_H + 10.0; + let half = (rect.width() - PAD * 2.0 - 10.0) / 2.0; + let natural = filmcraft_tts::catalog::find(&d.voice).is_some(); + let can_say = filmcraft_tts::check_text(&d.text).is_ok() && (!natural || installed) && !busy; + let pr = Rect::from_min_size(pos2(rect.min.x + PAD, fy), vec2(half, 28.0)); + if button(ui, &t, pr, Icon::Play, if busy { "Synthesizing…" } else { "Preview" }, false, can_say, "tts-preview").clicked() && can_say { + actions.push(Action::Run("tts.preview", d.params(), true)); + } + auto.add("tts.preview", pr, "Preview"); + let sr = Rect::from_min_size(pos2(pr.max.x + 10.0, fy), vec2(half, 28.0)); + let can_save = can_say && app.session.active_sequence().is_some(); + if button(ui, &t, sr, Icon::Plus, if editing.is_some() { "Save" } else { "Add to timeline" }, true, can_save, "tts-save").clicked() && can_save { + let mut p = d.params(); + match editing { + Some(c) => { + p["clip"] = json!(c); + actions.push(Action::Run("tts.edit", p, false)); + } + None => actions.push(Action::Run("tts.create", p, false)), + } + } + auto.add("tts.save", sr, "Save"); + + for a in actions { + match a { + Action::New => { + let keep = (app.ui.tts.language.clone(), app.ui.tts.voice.clone(), app.ui.tts.pitch, app.ui.tts.pace); + app.session.state.selection.clear(); + let dl = app.ui.tts.download_job; + app.ui.tts = TtsDraft { language: keep.0, voice: keep.1, pitch: keep.2, pace: keep.3, download_job: dl, ..Default::default() }; + } + Action::Run(cmd, p, play) => start(app, cmd, p, play), + Action::Download => { + app.ui.tts.confirm_download = false; + match app.session.execute("tts.downloadVoices", json!({})) { + Ok(r) => app.ui.tts.download_job = r["job"].as_u64(), + Err(e) => app.ui.status = e.to_string(), + } + } + Action::CancelDownload => { + if let Some(id) = app.ui.tts.download_job + && let Some(j) = app.session.jobs.iter().find(|j| j.id == id) + { + j.progress.cancel.store(true, std::sync::atomic::Ordering::Relaxed); + } + } + } + } +} + +/// The part of `r` the scroll area shows (what a click can reach). +fn visible(ui: &egui::Ui, r: Rect) -> Rect { + r.intersect(ui.clip_rect()) +} + +fn label(ui: &mut egui::Ui, t: &Tokens, text: &str) { + ui.label(egui::RichText::new(text).color(t.text).size(12.5).strong()); + ui.add_space(2.0); +} + +#[allow(clippy::too_many_arguments)] +fn button(ui: &mut egui::Ui, t: &Tokens, r: Rect, icon: Icon, text: &str, primary: bool, active: bool, id: &str) -> egui::Response { + let resp = ui.interact(r, egui::Id::new(id), Sense::click()); + let p = ui.painter(); + let (bg, fg) = match (primary, active) { + (true, true) => (if resp.hovered() { t.accent_hover } else { t.accent }, egui::Color32::WHITE), + (_, false) => (t.field_bg, t.text_faint), + (false, true) => (if resp.hovered() { t.hover } else { t.field_bg }, t.text), + }; + p.rect_filled(r, 4.0, bg); + if !primary { + p.rect_stroke(r, 4.0, Stroke::new(1.0, t.field_border), StrokeKind::Inside); + } + let g = p.layout_no_wrap(text.to_string(), Tokens::semibold(12.5), fg); + let total = 18.0 + g.size().x; + let x0 = r.center().x - total / 2.0; + icons::paint(p, Rect::from_center_size(pos2(x0 + 7.0, r.center().y), vec2(14.0, 14.0)), icon, fg); + p.galley(pos2(x0 + 18.0, r.center().y - g.size().y / 2.0), g, fg); + resp +} diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index a8f12b8a5..aacea877e 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -411,6 +411,9 @@ pub struct UiState { /// Essential Sound sub-tab: "Edit" or "Browse". #[serde(default)] pub essential_sound_tab: String, + /// Text to Speech panel: the script and voice settings being written or edited. + #[serde(default)] + pub tts: crate::panels::tts::TtsDraft, /// Export mode: settings, preset, destination, range, the Preset Manager and Quick Export. #[serde(default)] pub export: crate::panels::export_mode::ExportUi, @@ -454,6 +457,9 @@ pub struct UiState { /// Sequence Settings dialog draft. #[serde(default)] pub sequence_settings: SequenceSettingsDraft, + /// Set Transition Duration dialog draft (double-click a transition). + #[serde(default)] + pub transition_duration: TransitionDurationDraft, /// On-monitor text editing (Type tool / double-click on a text layer). #[serde(default)] pub gfx_edit: Option, @@ -757,6 +763,15 @@ impl Default for AddTracksDraft { } } +/// The Set Transition Duration dialog (double-click a transition in the Timeline): which +/// transition, and its duration in frames. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(default)] +pub struct TransitionDurationDraft { + pub transition: u64, + pub frames: i64, +} + /// The Sequence Settings dialog (Sequence ▸ Sequence Settings…), filled from the active sequence /// when it opens (`panels::sequence_settings::open`). `tab`: `general`, `color` or `vr`; the /// timebase is `fps_num`/`fps_den`; `mix`: `Stereo`, `Mono`, `5.1` or `Adaptive`; `working_space`: @@ -890,6 +905,7 @@ impl Default for UiState { status: String::new(), eyedropper: None, essential_sound_tab: "Edit".into(), + tts: Default::default(), export: Default::default(), text_tab: captions_tab(), caption_search: String::new(), @@ -904,6 +920,7 @@ impl Default for UiState { add_tracks: AddTracksDraft::default(), delete_tracks: DeleteTracksDraft::default(), sequence_settings: SequenceSettingsDraft::default(), + transition_duration: TransitionDurationDraft::default(), gfx_edit: None, pen_points: vec![], link_media: None, diff --git a/crates/ui-egui/tests/graphics_ui.rs b/crates/ui-egui/tests/graphics_ui.rs index b866b9bfd..c7aa5138e 100644 --- a/crates/ui-egui/tests/graphics_ui.rs +++ b/crates/ui-egui/tests/graphics_ui.rs @@ -152,3 +152,81 @@ fn type_tool_edit_style_and_move() { d.ok("ui.set", json!({"tool": "Selection"})); d.shot("shape-layer"); } + +/// The value of `param` on the text layer of graphic clip `clip`, at its first frame. +fn text_param(d: &mut Driver, clip: u64, param: &str) -> Option { + let s = &d.harness.state().session; + let (_, it) = s.active_sequence()?.find_item(filmcraft_project::ClipId(clip))?; + let e = it.effects.iter().find(|e| e.effect == "graphic_text")?; + Some(e.param(param)?.value_at(filmcraft_time::Tick::ZERO)) +} + +/// A drag of a property in the Properties panel is one undo step, so one Cmd+Z restores the +/// original value (a drag was committing one undo step per frame, so undo only stepped back one +/// frame's worth; the shadow color picker showed it as "sometimes undo works, sometimes not"). +#[test] +fn a_property_drag_is_one_undo_step() { + let mut d = Driver::demo(); + d.ok("ui.set", json!({"workspace": "Captions and Graphics"})); + let r = d.exec("graphics.newText", json!({"text": "Title"})); + let clip = r["clip"].as_u64().unwrap(); + d.exec("graphics.selectLayer", json!({"clip": clip, "layers": [0]})); + d.frames(4); + // the layer's Opacity number (visible in the Align and Transform section) + d.exec("graphics.set", json!({"clip": clip, "layer": 0, "props": {"opacity": 50}})); + d.frames(2); + let opacity = |d: &mut Driver| -> f64 { text_param(d, clip, "opacity").and_then(|v| v.as_f64()).unwrap() }; + let before = opacity(&mut d); + let undo0 = d.exec("history.list", json!({}))["undo"].as_array().unwrap().len(); + // drag the Opacity number to the left: several move frames, one undo step + let n = d.rect("graphics.prop.opacity"); + let (cx, cy) = (n[0] + n[2] / 2.0, n[1] + n[3] / 2.0); + d.ok("ui.drag", json!({"from": {"x": cx, "y": cy}, "to": {"x": cx - 60.0, "y": cy}, "steps": 6})); + d.frames(4); + let after = opacity(&mut d); + assert!(after < before - 5.0, "the drag changed the opacity: {before} -> {after}"); + let undo1 = d.exec("history.list", json!({}))["undo"].as_array().unwrap().len(); + assert_eq!(undo1, undo0 + 1, "a property drag is one undo step"); + d.exec("edit.undo", json!({})); + assert_eq!(opacity(&mut d), before, "one undo restores the original value"); +} + +/// The shadow color picker popup: a drag in the 2D picker is one undo step, so one Cmd+Z +/// restores the original color (a drag was committing one undo step per frame, so undo only +/// stepped back one frame's worth — the reported "sometimes undo works, sometimes not"). +#[test] +fn shadow_color_drag_is_one_undo_step() { + let mut d = Driver::demo(); + d.ok("ui.set", json!({"workspace": "Captions and Graphics"})); + let r = d.exec("graphics.newText", json!({"text": "Title"})); + let clip = r["clip"].as_u64().unwrap(); + d.exec("graphics.selectLayer", json!({"clip": clip, "layers": [0]})); + d.frames(4); + let color = |d: &mut Driver| text_param(d, clip, "shadow_color").and_then(|v| v.as_color()); + // scroll the properties panel so the shadow row is visible, then open the picker + d.ok("ui.scroll", json!({"x": 1400.0, "y": 500.0, "dx": 0.0, "dy": -400.0})); + d.frames(3); + let sw = d.rect("graphics.prop.shadow_color"); + d.ok("ui.click", json!({"x": sw[0] + sw[2] / 2.0, "y": sw[1] + sw[3] / 2.0})); + d.frames(3); + // the popup flips above the swatch (it does not fit below); find a point in its 2D picker + let undo0 = d.exec("history.list", json!({}))["undo"].as_array().unwrap().len(); + let mut hit = None; + for (dx, dy) in [(30.0, 30.0), (150.0, 30.0), (30.0, 150.0), (150.0, 150.0)] { + let (px, py) = (sw[0] + dx, sw[1] - dy); + let before = color(&mut d); + d.ok("ui.drag", json!({"from": {"x": px, "y": py}, "to": {"x": px + 60.0, "y": py + 40.0}, "steps": 4})); + d.frames(3); + let after = color(&mut d); + if after != before { + hit = Some((before, after)); + break; + } + } + let (before, after) = hit.expect("no drag point hit the color picker"); + assert_ne!(after, before, "the drag changed the shadow color"); + let undo1 = d.exec("history.list", json!({}))["undo"].as_array().unwrap().len(); + assert_eq!(undo1, undo0 + 1, "a color drag is one undo step"); + d.exec("edit.undo", json!({})); + assert_eq!(color(&mut d), before, "one undo restores the original color"); +} diff --git a/crates/ui-egui/tests/playback_clock_ui.rs b/crates/ui-egui/tests/playback_clock_ui.rs index 81570d161..3fb7b3e67 100644 --- a/crates/ui-egui/tests/playback_clock_ui.rs +++ b/crates/ui-egui/tests/playback_clock_ui.rs @@ -16,6 +16,7 @@ struct FakeOut { played: Arc, per_poll: u64, started: bool, + note: Option, } impl AudioOut for FakeOut { @@ -33,6 +34,9 @@ impl AudioOut for FakeOut { fn played_frames(&self) -> Option { self.started.then(|| self.played.fetch_add(self.per_poll, Ordering::SeqCst) + self.per_poll) } + fn note(&self) -> Option { + self.note.clone() + } } /// Play for `steps` × 0.25 s of simulated time with the given output; the playhead in seconds. @@ -41,7 +45,7 @@ fn play_with(per_poll: u64, steps: usize) -> (f64, bool) { s.execute("file.openDemoProject", json!({})).unwrap(); s.set_playhead(Tick::ZERO); let mut app = FilmcraftApp::new(s); - app.audio = Some(Box::new(FakeOut { played: Arc::new(AtomicU64::new(0)), per_poll, started: false })); + app.audio = Some(Box::new(FakeOut { played: Arc::new(AtomicU64::new(0)), per_poll, started: false, note: None })); let mut h = Harness::builder().with_size(egui::vec2(1280.0, 800.0)).with_step_dt(0.25).build_eframe(move |_cc| app); h.step(); h.state_mut().play(1.0); @@ -68,3 +72,23 @@ fn a_playing_audio_device_drives_the_clock() { assert!(t > 0.2, "{t:.2} s"); assert!(audio_clock); } + +/// A fallback note from the audio device (broken ALSA `default`, #23/#106) reaches the status +/// bar on the first frame, and survives while playback runs. +#[test] +fn a_fallback_note_from_the_audio_device_reaches_the_status_bar() { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.set_playhead(Tick::ZERO); + let note = "Audio hardware: the default output device cannot be opened; using 'sofhdadsp'".to_string(); + let mut app = FilmcraftApp::new(s); + app.audio = Some(Box::new(FakeOut { played: Arc::new(AtomicU64::new(0)), per_poll: 4_800, started: false, note: Some(note.clone()) })); + let mut h = Harness::builder().with_size(egui::vec2(1280.0, 800.0)).with_step_dt(0.25).build_eframe(move |_cc| app); + h.step(); + assert_eq!(h.state_mut().ui.status, note, "the note surfaces on the first frame (apply_prefs)"); + h.state_mut().play(1.0); + for _ in 0..16 { + h.step(); + } + assert_eq!(h.state_mut().ui.status, note, "the note survives while playback runs"); +} diff --git a/crates/ui-egui/tests/timeline_drop_ui.rs b/crates/ui-egui/tests/timeline_drop_ui.rs new file mode 100644 index 000000000..76bdde9fa --- /dev/null +++ b/crates/ui-egui/tests/timeline_drop_ui.rs @@ -0,0 +1,154 @@ +//! Files dropped from the OS onto the Timeline. With Preferences ▸ Timeline ▸ "Place files dropped +//! onto the Timeline directly on the Timeline" (`drop_import_to_timeline`) they are also placed at +//! the pointer; with it off they are only imported to the bin. + +use std::path::{Path, PathBuf}; + +use egui_kittest::Harness; +use filmcraft_engine::Session; +use filmcraft_ui_egui::FilmcraftApp; +use serde_json::json; + +fn step(h: &mut Harness<'static, FilmcraftApp>) { + let ctx = h.ctx.clone(); + let mut raw = std::mem::take(h.input_mut()); + eframe::App::raw_input_hook(h.state_mut(), &ctx, &mut raw); + *h.input_mut() = raw; + ctx.request_repaint(); + h.step(); +} + +fn harness() -> Harness<'static, FilmcraftApp> { + let mut s = Session::default(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("file.newSequence", json!({"name":"Drop test","width":1280,"height":720,"fps":24})).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).build_eframe(move |_cc| FilmcraftApp::new(s)); + for _ in 0..4 { + step(&mut h); + } + h +} + +fn pointer_to(h: &mut Harness<'static, FilmcraftApp>, pos: egui::Pos2) { + h.input_mut().events.push(egui::Event::PointerMoved(pos)); + step(h); +} + +fn control_rect(h: &Harness<'static, FilmcraftApp>, id: &str) -> egui::Rect { + let [x, y, w, height] = h.state().auto.elements.iter().find(|e| e.id == id).or_else(|| h.state().auto.find(id)).unwrap().rect; + egui::Rect::from_min_size(egui::pos2(x, y), egui::vec2(w, height)) +} + +/// A point 60 px into the Timeline's clip area on the named track row. +fn point_on_row(h: &Harness<'static, FilmcraftApp>, row_control: &str) -> egui::Pos2 { + let row = control_rect(h, row_control); + let panel = control_rect(h, "panel.Timeline"); + egui::pos2(panel.min.x + h.state().ui.timeline.header_w + 60.0, row.center().y) +} + +/// A file the OS dropped on the window, as the native integration hands it to egui. +#[derive(Debug)] +struct OsFile(PathBuf); + +impl egui::DroppedFile for OsFile { + fn path(&self) -> &Path { + &self.0 + } + + fn bytes(&self) -> Result, String> { + std::fs::read(&self.0).map_err(|e| e.to_string()) + } +} + +/// Drops `path` with the pointer over `row_control`, as the OS does: the pointer is hovering first, +/// then one frame carries the dropped file. The file is cleared afterwards, as egui only reports it once. +fn drop_on_row(h: &mut Harness<'static, FilmcraftApp>, row_control: &str, path: &Path) { + let at = point_on_row(h, row_control); + pointer_to(h, at); + h.input_mut().dropped_files.push(std::sync::Arc::new(OsFile(path.to_path_buf()))); + step(h); + h.input_mut().dropped_files.clear(); + step(h); +} + +/// (video clips, audio clips) on the active sequence. +fn clip_counts(h: &Harness<'static, FilmcraftApp>) -> (usize, usize) { + let q = h.state().session.active_sequence().unwrap(); + (q.video_tracks.iter().map(|t| t.items.len()).sum(), q.audio_tracks.iter().map(|t| t.items.len()).sum()) +} + +fn fixture(name: &str) -> PathBuf { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/fixtures/ui-egui"); + std::fs::create_dir_all(&dir).unwrap(); + dir.join(name) +} + +/// A small generated PNG still. +fn still(name: &str) -> PathBuf { + let p = fixture(name); + image::RgbaImage::from_pixel(64, 36, image::Rgba([40, 140, 220, 255])).save(&p).unwrap(); + p +} + +/// A 0.5 s mono 48 kHz 16-bit WAV tone (audio only). +fn tone(name: &str) -> PathBuf { + let p = fixture(name); + let rate = 48_000u32; + let n = rate / 2; + let mut data = Vec::with_capacity(n as usize * 2); + for i in 0..n { + let s = (f32::sin(2.0 * std::f32::consts::PI * 440.0 * i as f32 / rate as f32) * 0.5 * i16::MAX as f32) as i16; + data.extend_from_slice(&s.to_le_bytes()); + } + let mut wav = Vec::with_capacity(44 + data.len()); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(36 + data.len() as u32).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16u32.to_le_bytes()); + wav.extend_from_slice(&1u16.to_le_bytes()); // PCM + wav.extend_from_slice(&1u16.to_le_bytes()); // mono + wav.extend_from_slice(&rate.to_le_bytes()); + wav.extend_from_slice(&(rate * 2).to_le_bytes()); + wav.extend_from_slice(&2u16.to_le_bytes()); + wav.extend_from_slice(&16u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&(data.len() as u32).to_le_bytes()); + wav.extend_from_slice(&data); + std::fs::write(&p, wav).unwrap(); + p +} + +#[test] +fn still_dropped_on_the_video_row_is_placed_at_the_pointer() { + let mut h = harness(); + let items_before = h.state().session.project.items.len(); + drop_on_row(&mut h, "timeline.track.V1.target", &still("drop_still_video_row.png")); + assert_eq!(h.state().session.project.items.len(), items_before + 1, "the still is imported to the bin"); + assert_eq!(clip_counts(&h), (1, 0), "status: {}", h.state().ui.status); +} + +#[test] +fn still_dropped_on_an_audio_row_goes_to_the_video_track_without_error() { + // a video-only item over an audio row: the picture takes the matching video track, no error + let mut h = harness(); + drop_on_row(&mut h, "timeline.track.A1.target", &still("drop_still_audio_row.png")); + assert_eq!(clip_counts(&h), (1, 0), "status: {}", h.state().ui.status); +} + +#[test] +fn tone_dropped_on_a_video_row_goes_to_the_audio_track_without_error() { + // an audio-only item over a video row: the sound takes the matching audio track, no error + let mut h = harness(); + drop_on_row(&mut h, "timeline.track.V1.target", &tone("drop_tone_video_row.wav")); + assert_eq!(clip_counts(&h), (0, 1), "status: {}", h.state().ui.status); +} + +#[test] +fn with_drop_import_off_the_file_is_only_imported() { + let mut h = harness(); + h.state_mut().session.prefs.timeline.drop_import_to_timeline = false; + let items_before = h.state().session.project.items.len(); + drop_on_row(&mut h, "timeline.track.V1.target", &still("drop_still_pref_off.png")); + assert_eq!(h.state().session.project.items.len(), items_before + 1, "the still is imported to the bin"); + assert_eq!(clip_counts(&h), (0, 0), "status: {}", h.state().ui.status); +} diff --git a/crates/ui-egui/tests/transitions_ui.rs b/crates/ui-egui/tests/transitions_ui.rs new file mode 100644 index 000000000..fbcbabb57 --- /dev/null +++ b/crates/ui-egui/tests/transitions_ui.rs @@ -0,0 +1,276 @@ +//! Headless UI tests of applied transitions in the Timeline (#430, #224): click one to select it +//! and see it in Effect Controls, Delete it, drag its end or its middle, double-click it for Set +//! Transition Duration. The real `FilmcraftApp` under `egui_kittest`, driven over the control +//! channel by automation id. +//! +//! Set `FILMCRAFT_UI_SNAPSHOT_DIR=` to also render the window offscreen with wgpu and write +//! `transition-*.png` there; without it no GPU is needed. + +use std::sync::mpsc::{Sender, channel}; + +use egui_kittest::Harness; +use filmcraft_engine::Session; +use filmcraft_ui_egui::FilmcraftApp; +use filmcraft_ui_egui::control::ControlRequest; +use serde_json::{Value, json}; + +struct Driver { + harness: Harness<'static, FilmcraftApp>, + tx: Sender, + snapshots: Option, +} + +/// The demo's first video transition between two clips. +#[derive(Clone, Copy, Debug)] +struct Crossing { + id: u64, + start: i64, + duration: i64, + cut: i64, +} + +impl Driver { + fn demo() -> Self { + let mut session = Session::default(); + session.execute("file.openDemoProject", json!({})).expect("demo project"); + let (tx, rx) = channel(); + let app = FilmcraftApp::new(session).with_control(rx); + let snapshots = std::env::var_os("FILMCRAFT_UI_SNAPSHOT_DIR").map(std::path::PathBuf::from); + // 60 fps steps, so a double-click's two clicks fall within egui's double-click time + let mut b = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_step_dt(1.0 / 60.0).with_max_steps(10_000); + if snapshots.is_some() { + b = b.wgpu(); + } + let harness = b.build_eframe(move |_cc| app); + let mut d = Driver { harness, tx, snapshots }; + d.frames(4); + d.ok("ui.panel.show", json!({"panel": "EffectControls"})); + d.frames(3); + d + } + + fn frames(&mut self, n: usize) { + for _ in 0..n { + let ctx = self.harness.ctx.clone(); + let mut raw = std::mem::take(self.harness.input_mut()); + eframe::App::raw_input_hook(self.harness.state_mut(), &ctx, &mut raw); + *self.harness.input_mut() = raw; + self.harness.step(); + } + } + + fn call(&mut self, method: &str, params: Value) -> Value { + let (req, reply) = ControlRequest::new(method, params.clone()); + self.tx.send(req).unwrap(); + for _ in 0..600 { + self.frames(1); + if let Ok(v) = reply.try_recv() { + return v; + } + } + panic!("no reply to {method} {params}"); + } + + fn ok(&mut self, method: &str, params: Value) -> Value { + let v = self.call(method, params.clone()); + assert_eq!(v["ok"], json!(true), "{method} {params} failed: {v}"); + v["result"].clone() + } + + fn exec(&mut self, command: &str, params: Value) -> Value { + self.ok("engine.execute", json!({"command": command, "params": params})) + } + + fn find(&mut self, id: &str) -> Option<(f64, f64, f64, f64, String)> { + self.frames(2); + let v = self.ok("ui.elements", json!({"prefix": id})); + let e = v.as_array().unwrap().iter().find(|e| e["id"] == id)?.clone(); + let r: Vec = e["rect"].as_array().unwrap().iter().map(|x| x.as_f64().unwrap()).collect(); + Some((r[0], r[1], r[2], r[3], e["label"].as_str().unwrap_or_default().to_string())) + } + + /// With `FILMCRAFT_UI_SNAPSHOT_DIR` set, render the whole window offscreen to `.png`. + fn snapshot(&mut self, name: &str) { + let Some(dir) = self.snapshots.clone() else { return }; + self.frames(2); + match self.harness.render() { + Ok(img) => { + std::fs::create_dir_all(&dir).unwrap(); + img.save(dir.join(format!("{name}.png"))).unwrap(); + } + Err(e) => eprintln!("snapshot {name} skipped: {e}"), + } + } + + fn label(&mut self, id: &str) -> String { + self.find(id).unwrap_or_else(|| panic!("no element {id}")).4 + } + + fn click(&mut self, id: &str) { + self.find(id).unwrap_or_else(|| panic!("no element {id}")); + self.ok("ui.click", json!({"id": id})); + self.frames(3); + } + + fn crossing(&mut self) -> Crossing { + let q = self.exec("sequence.inspect", json!({})); + for tr in q["video"].as_array().unwrap() { + for x in tr["transitions"].as_array().unwrap() { + if let (Some(_), Some(to)) = (x["from"].as_u64(), x["to"].as_u64()) { + let cut = tr["items"].as_array().unwrap().iter().find(|i| i["clip"] == to).unwrap()["start"].as_i64().unwrap(); + return Crossing { id: x["id"].as_u64().unwrap(), start: x["start"].as_i64().unwrap(), duration: x["duration"].as_i64().unwrap(), cut }; + } + } + } + panic!("the demo has a transition between two clips"); + } + + fn transition(&mut self, id: u64) -> Option { + let q = self.exec("sequence.inspect", json!({})); + q["video"].as_array().unwrap().iter().flat_map(|t| t["transitions"].as_array().unwrap().clone()).find(|x| x["id"] == id) + } + + fn rate(&mut self) -> (i64, i64) { + let q = self.exec("sequence.inspect", json!({})); + (q["settings"]["frame_rate"]["num"].as_i64().unwrap(), q["settings"]["frame_rate"]["den"].as_i64().unwrap()) + } +} + +/// Ticks in one frame of `rate`. +fn frame_ticks((num, den): (i64, i64)) -> i64 { + 254_016_000_000 * den / num +} + +/// Clicking a transition selects it (and only it) and opens it in Effect Controls; Delete removes +/// it, leaving its clips; undo brings it back. +#[test] +fn click_selects_a_transition_and_delete_removes_it() { + let mut d = Driver::demo(); + let c = d.crossing(); + let tid = format!("timeline.transition.{}", c.id); + d.click(&tid); + let q = d.exec("sequence.inspect", json!({})); + assert_eq!(q["transitionSelection"], json!([c.id]), "{q}"); + assert_eq!(q["selection"], json!([]), "a transition replaces the clip selection"); + // Effect Controls shows it: duration as timecode, its alignment, Reverse + let (num, den) = d.rate(); + let rate = filmcraft_engine::time::FrameRate { num, den }; + let frames = c.duration / frame_ticks((num, den)); + assert_eq!(d.label("effectControls.transition.duration"), filmcraft_engine::time::format_timecode_frames(frames, rate, false)); + assert_eq!(d.label("effectControls.transition.alignment"), "Center at Cut"); + assert_eq!(d.label("effectControls.transition.reverse"), "false"); + d.snapshot("transition-selected"); + let clips_before = q["video"].as_array().unwrap().iter().map(|t| t["items"].as_array().unwrap().len()).sum::(); + // Delete + d.ok("ui.key", json!({"key": "Delete"})); + d.frames(3); + assert!(d.transition(c.id).is_none(), "Delete removes the selected transition"); + let q = d.exec("sequence.inspect", json!({})); + assert_eq!(q["video"].as_array().unwrap().iter().map(|t| t["items"].as_array().unwrap().len()).sum::(), clips_before, "its clips stay"); + assert!(d.find("effectControls.transition.duration").is_none(), "Effect Controls lets go of it"); + d.exec("edit.undo", json!({})); + assert!(d.transition(c.id).is_some(), "undo brings it back"); + // clicking a clip selects the clip instead + let clip = q["video"][0]["items"][0]["clip"].as_u64().unwrap(); + d.exec("timeline.select", json!({"transitions": [c.id]})); + d.exec("timeline.select", json!({"clips": [clip]})); + assert_eq!(d.exec("sequence.inspect", json!({}))["transitionSelection"], json!([])); +} + +/// Dragging a transition's end trims that end only (#224); dragging its middle slides it over the +/// cut (Custom Start). Each drag is one undo step. +#[test] +fn dragging_a_transition_end_or_middle() { + let mut d = Driver::demo(); + let c = d.crossing(); + let frame = frame_ticks(d.rate()); + let undo_before = d.exec("sequence.inspect", json!({})); + let (x, y, w, h, _) = d.find(&format!("timeline.transition.{}.out", c.id)).expect("the transition's end"); + let (fx, fy) = (x + w / 2.0, y + h / 2.0); + d.ok("ui.drag", json!({"from": {"x": fx, "y": fy}, "to": {"x": fx + 40.0, "y": fy}, "steps": 8})); + d.frames(3); + let x1 = d.transition(c.id).unwrap(); + let (s1, d1) = (x1["start"].as_i64().unwrap(), x1["duration"].as_i64().unwrap()); + assert!(d1 > c.duration, "dragging the end out lengthens it: {} -> {d1}", c.duration); + assert_eq!(s1, c.start, "the start stays where it was"); + assert_eq!((d1 - c.duration) % frame, 0, "by whole frames"); + assert_eq!(x1["align"], json!("custom"), "no longer centred: Custom Start"); + // the middle: slide it right of centre + let (x, y, w, h, _) = d.find(&format!("timeline.transition.{}", c.id)).unwrap(); + let (mx, my) = (x + w / 2.0, y + h / 2.0); + // more than egui's 6 px click tolerance, or the press is a click + d.ok("ui.drag", json!({"from": {"x": mx, "y": my}, "to": {"x": mx + 14.0, "y": my}, "steps": 6})); + d.frames(3); + let x2 = d.transition(c.id).unwrap(); + assert_eq!(x2["duration"].as_i64(), Some(d1), "sliding keeps the duration"); + assert!(x2["start"].as_i64().unwrap() > s1, "and moves it right"); + assert!(x2["start"].as_i64().unwrap() <= c.cut, "still over the cut"); + // off-centre (Custom Start), or Start at Cut if the slide reached the cut, where it stops + assert!(matches!(x2["align"].as_str(), Some("custom" | "start")), "{}", x2["align"]); + // two drags, two undo steps + d.exec("edit.undo", json!({})); + d.exec("edit.undo", json!({})); + let back = d.transition(c.id).unwrap(); + assert_eq!((back["start"].as_i64(), back["duration"].as_i64()), (Some(c.start), Some(c.duration)), "{undo_before}"); +} + +/// Double-clicking a transition opens Set Transition Duration, whose OK sets the duration +/// (following the alignment); Effect Controls' Alignment menu re-aligns it on the cut. +#[test] +fn set_transition_duration_and_alignment_menu() { + let mut d = Driver::demo(); + let c = d.crossing(); + let frame = frame_ticks(d.rate()); + // double-click it + let (x0, y0, w, h, _) = d.find(&format!("timeline.transition.{}", c.id)).unwrap(); + d.ok("ui.click", json!({"x": x0 + w / 2.0, "y": y0 + h / 2.0, "count": 2})); + d.frames(3); + let frames = c.duration / frame; + assert!(d.find("transitionDuration.value").is_some(), "Set Transition Duration opened"); + assert_eq!(d.harness.state().ui.transition_duration.frames, frames, "with its duration"); + assert_eq!(d.exec("sequence.inspect", json!({}))["transitionSelection"], json!([c.id]), "and selected it"); + d.snapshot("transition-duration-dialog"); + // Center at Cut: a new duration moves both ends + d.harness.state_mut().ui.transition_duration.frames = 10; + d.frames(2); + d.click("transitionDuration.ok"); + assert!(d.find("transitionDuration.value").is_none(), "closed"); + let x = d.transition(c.id).unwrap(); + assert_eq!((x["start"].as_i64(), x["duration"].as_i64(), x["align"].as_str()), (Some(c.cut - 5 * frame), Some(10 * frame), Some("center"))); + // the Alignment menu: Start at Cut + d.click("effectControls.transition.alignment"); + d.click("effectControls.transition.alignment.option.start"); + let x = d.transition(c.id).unwrap(); + assert_eq!((x["start"].as_i64(), x["duration"].as_i64(), x["align"].as_str()), (Some(c.cut), Some(10 * frame), Some("start"))); + assert_eq!(d.label("effectControls.transition.alignment"), "Start at Cut"); + // Reverse + d.click("effectControls.transition.reverse"); + assert_eq!(d.transition(c.id).unwrap()["reverse"], json!(true)); +} + +/// A transition's own settings are in Effect Controls and change it (Push: Direction, Motion Blur). +#[test] +fn transition_settings_in_effect_controls() { + let mut d = Driver::demo(); + let q = d.exec("sequence.inspect", json!({})); + let push = q["video"] + .as_array() + .unwrap() + .iter() + .flat_map(|t| t["transitions"].as_array().unwrap().clone()) + .find(|x| x["effect"] == "push") + .expect("the demo has a Push"); + let id = push["id"].as_u64().unwrap(); + d.click(&format!("timeline.transition.{id}")); + for p in ["direction", "motion_blur"] { + assert!(d.find(&format!("effectControls.transition.param.{p}")).is_some(), "{p} shown"); + } + d.snapshot("transition-push-settings"); + d.click("effectControls.transition.param.direction"); + d.click("effectControls.transition.param.direction.option.0"); + assert_eq!(d.transition(id).unwrap()["params"]["direction"], json!({"Choice": 0})); + d.click("effectControls.transition.reset"); + let def = filmcraft_engine::project::vtransition::find_transition("push", filmcraft_engine::project::EffectKind::VideoTransition).unwrap(); + let want = serde_json::to_value(&def.instance().params.get("direction").unwrap().value).unwrap(); + assert_eq!(d.transition(id).unwrap()["params"]["direction"], want, "Reset puts Push's own defaults back"); +} diff --git a/crates/ui-egui/tests/tts_ui.rs b/crates/ui-egui/tests/tts_ui.rs new file mode 100644 index 000000000..4a7576285 --- /dev/null +++ b/crates/ui-egui/tests/tts_ui.rs @@ -0,0 +1,205 @@ +//! Headless UI tests of the Text to Speech panel: typing a script, Add pause, adding the narration +//! to the timeline, clicking the narration clip to load it back, and saving an edit in place. + +use std::sync::mpsc::{Sender, channel}; + +use egui_kittest::Harness; +use filmcraft_engine::Session; +use filmcraft_ui_egui::FilmcraftApp; +use filmcraft_ui_egui::control::ControlRequest; +use serde_json::{Value, json}; + +struct Driver { + harness: Harness<'static, FilmcraftApp>, + tx: Sender, +} + +impl Driver { + fn demo(scratch: &str) -> Self { + let mut session = Session::default(); + session.execute("file.openDemoProject", json!({})).expect("demo project"); + session.execute("file.projectSettings.scratchDisks", json!({"captured": scratch})).expect("scratch disk"); + let (tx, rx) = channel(); + let app = FilmcraftApp::new(session).with_control(rx); + let harness = Harness::builder().with_size(egui::vec2(1600.0, 980.0)).with_max_steps(10_000).build_eframe(move |_cc| app); + let mut d = Driver { harness, tx }; + d.frames(4); + d + } + + fn frames(&mut self, n: usize) { + for _ in 0..n { + let ctx = self.harness.ctx.clone(); + let mut raw = std::mem::take(self.harness.input_mut()); + eframe::App::raw_input_hook(self.harness.state_mut(), &ctx, &mut raw); + *self.harness.input_mut() = raw; + self.harness.step(); + } + } + + fn call(&mut self, method: &str, params: Value) -> Value { + let (req, reply) = ControlRequest::new(method, params.clone()); + self.tx.send(req).unwrap(); + for _ in 0..600 { + self.frames(1); + if let Ok(v) = reply.try_recv() { + return v; + } + } + panic!("no reply to {method} {params}"); + } + + fn ok(&mut self, method: &str, params: Value) -> Value { + let v = self.call(method, params.clone()); + assert_eq!(v["ok"], json!(true), "{method} {params} failed: {v}"); + v["result"].clone() + } + + fn exec(&mut self, command: &str, params: Value) -> Value { + self.ok("engine.execute", json!({"command": command, "params": params})) + } + + fn click(&mut self, id: &str) { + self.ok("ui.click", json!({"id": id})); + self.frames(3); + } + + fn type_text(&mut self, text: &str) { + self.ok("ui.type", json!({"text": text})); + self.frames(3); + } + + fn ui(&mut self) -> Value { + self.ok("ui.inspect", json!({})) + } + + fn draft(&mut self) -> Value { + self.ui()["ui"]["tts"].clone() + } + + /// Let a background synthesis job finish. + fn settle(&mut self) { + for _ in 0..400 { + if self.draft()["pending"].is_null() { + return; + } + std::thread::sleep(std::time::Duration::from_millis(10)); + self.frames(1); + } + panic!("synthesis never finished"); + } + + fn ids(&mut self, prefix: &str) -> Vec { + let v = self.ok("ui.elements", json!({"prefix": prefix})); + v.as_array().unwrap().iter().filter_map(|e| e["id"].as_str().map(str::to_string)).collect() + } + + fn narration_clips(&mut self) -> Vec { + let seq = self.exec("sequence.inspect", json!({})); + let mut out = Vec::new(); + for t in seq["audio"].as_array().into_iter().flatten() { + for it in t["items"].as_array().into_iter().flatten() { + let id = it["clip"].as_u64().unwrap(); + if self.call("engine.execute", json!({"command": "tts.inspect", "params": {"clip": id}}))["ok"] == json!(true) { + out.push(id); + } + } + } + out + } +} + +fn scratch(name: &str) -> String { + let d = std::env::temp_dir().join(format!("filmcraft-tts-ui-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d.to_string_lossy().into_owned() +} + +#[test] +fn write_add_click_to_edit_and_save_in_place() { + let dir = scratch("flow"); + let mut d = Driver::demo(&dir); + d.exec("window.panel.TextToSpeech", json!({})); + d.frames(3); + // the panel opens beside Properties, leaving the Program monitor visible + assert!(!d.ids("program.").is_empty(), "Program monitor still shown"); + let ids = d.ids("tts."); + for id in ["tts.language", "tts.voice", "tts.hearVoice", "tts.advanced", "tts.pitch", "tts.pace", "tts.text", "tts.addPause", "tts.preview", "tts.save"] { + assert!(ids.iter().any(|i| i == id), "missing {id} in {ids:?}"); + } + // type a script and add a pause + d.click("tts.text"); + d.type_text("Hello from the panel."); + d.click("tts.addPause"); + d.click("tts.text"); + d.ok("ui.key", json!({"key": "End"})); + d.type_text("Goodbye."); + let text = d.draft()["text"].as_str().unwrap().to_string(); + assert!(text.starts_with("Hello from the panel. [pause 1s]") && text.ends_with("Goodbye."), "{text:?}"); + // add it to the timeline + d.exec("playhead.set", json!({"seconds": 1.0})); + d.click("tts.save"); + d.settle(); + let clips = d.narration_clips(); + let status = d.ui()["ui"]["status"].clone(); + assert_eq!(clips.len(), 1, "one narration clip; status {status}"); + let clip = clips[0]; + let n = d.exec("tts.inspect", json!({"clip": clip})); + assert_eq!(n["narration"]["text"].as_str().unwrap(), text); + assert_eq!(d.ui()["selection"], json!([clip]), "the new clip is selected"); + assert_eq!(d.draft()["loadedFrom"][0], json!(clip), "the panel now edits it"); + // deselect and start fresh, then click the clip on the timeline: the panel loads it back + d.exec("edit.deselectAll", json!({})); + d.frames(3); + assert!(d.draft()["loadedFrom"].is_null()); + d.click("tts.text"); + d.ok("ui.key", json!({"key": "Cmd+A"})); + d.type_text("Something else"); + let at = d.ok("ui.timeline.locate", json!({"clip": clip})); + d.ok("ui.click", json!({"x": at["x"], "y": at["y"]})); + d.frames(4); + assert_eq!(d.ui()["selection"], json!([clip])); + assert_eq!(d.draft()["text"].as_str().unwrap(), text, "clicking the clip loads its script"); + // change the script and save: same clip, new audio, one undo step + let before = d.exec("tts.inspect", json!({"clip": clip}))["item"].clone(); + d.click("tts.text"); + d.ok("ui.key", json!({"key": "Cmd+A"})); + d.type_text("A new script for the same clip."); + d.click("tts.save"); + d.settle(); + let after = d.exec("tts.inspect", json!({"clip": clip})); + assert_eq!(after["narration"]["text"], "A new script for the same clip."); + assert_ne!(after["item"], before); + assert_eq!(d.narration_clips(), vec![clip]); + let hist = d.exec("history.list", json!({})); + assert_eq!(hist["undo"].as_array().unwrap().last().unwrap(), "Edit Narration"); + d.exec("edit.undo", json!({})); + d.frames(3); + assert_eq!(d.exec("tts.inspect", json!({"clip": clip}))["narration"]["text"].as_str().unwrap(), text); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn settings_controls_and_hear_this_voice() { + let dir = scratch("controls"); + let mut d = Driver::demo(&dir); + d.exec("window.panel.TextToSpeech", json!({})); + d.frames(3); + // Save is disabled with an empty script + d.click("tts.save"); + assert!(d.narration_clips().is_empty()); + // Hear this voice runs a preview (the headless harness has no audio output: a status line) + d.click("tts.hearVoice"); + d.settle(); + let status = d.ui()["ui"]["status"].as_str().unwrap_or_default().to_string(); + assert!(status.contains("audio output") || status.is_empty(), "{status}"); + // Advanced folds away and back + assert_eq!(d.draft()["advancedOpen"], true); + d.click("tts.advanced"); + assert_eq!(d.draft()["advancedOpen"], false); + assert!(!d.ids("tts.").iter().any(|i| i == "tts.pitch")); + d.click("tts.advanced"); + assert!(d.ids("tts.").iter().any(|i| i == "tts.pitch")); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/docs/agents.md b/docs/agents.md index 39d53f770..2e303cbb5 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -61,7 +61,7 @@ claude mcp add filmcraft -- filmcraft-cli mcp | `render_preview` | both | same as `render_frame` | | `project_inspect` | both | bins and items with ids, types, durations; active sequence | | `sequence_inspect` | both | the active sequence: tracks, clips (`start` / `end` and `sourceIn` / `sourceOut` in ticks, frames, `speed`, `reverse`: the clip plays its `sourceIn` to `sourceOut` stretch backward, `gainDb`), effects, transitions, markers (name and `comment`), playhead, selection | -| `media_import` | both | import files by absolute path (`text`, one path per line) | +| `media_import` | both | import files by absolute path (`paths`, an array; optional `bin`, `image_sequence`) | | `render_frame` | both | PNG of the program frame at `seconds` (headless renders; bridge screenshots the Program monitor). Read-only: the playhead and selection are left as they were | | `ui_inspect` | bridge | UI state: tool, workspace, panels, zoom, playback, fps | | `ui_elements` | bridge | on-screen interactive elements with id, label and rect (`prefix` filter) | @@ -182,7 +182,8 @@ Notes: ## 2b. Command-line interface Every command is also one shell call away. Options go anywhere; output is JSON; exit status is 0 on -success, 1 when a command fails and 2 on a usage error. `filmcraft-cli help` prints the reference. +success, 1 when a command fails and 2 on a usage error. `filmcraft-cli help`, `filmcraft-cli --help` +and `filmcraft-cli -h` print the reference and exit successfully. If the reader of stdout closes the pipe early (`filmcraft-cli commands | head`), the CLI drops the rest of its output but still finishes the work, saves included, and the exit status still reports failures; any other stdout write error is reported and makes the status 1. diff --git a/docs/architecture.md b/docs/architecture.md index d82099165..9481641f6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -24,7 +24,7 @@ Design principles: L5 ui-egui · automation · platform L4 engine L3 render · gpu · export · golden (test-only) - L2 edit · codecs · interchange · captions · speech + L2 edit · codecs · interchange · captions · speech · tts · tts-text L1 frame · media · project · audio-dsp · text L0 foundation: time · geom · color · bitstream · testkit (dev-dependency only) codecs/containers: isobmff · matroska · mxf · cfb · mpegts · ogg · h264 · h264enc · hevc · vp9 · av1 · mpeg2v · prores · dnx · apv · aac · ac3 · opus @@ -62,6 +62,8 @@ and `filmcraft-cli`. | `text` | L1 | text engine: font database (bundled OFL fonts + system fonts), shaping (harfrust), bidi, line breaking, paragraph layout, glyph/path rasteriser, strokes ([crates/text/README.md](../crates/text/README.md)) | | `edit` | L2 | pure edit algebra (insert, overwrite, razor, ripple, roll, slip, slide, rate stretch…; text-based editing: `edit::transcript`) | | `speech` | L2 | speech-to-text: `Transcriber` trait, Whisper model catalogue + verified downloader (feature `download`), pure-Rust Whisper inference on candle with word timestamps (feature `whisper`), speaker labelling ([transcripts.md](transcripts.md)) | +| `tts` | L2 | text to speech: `Voice` trait, voice catalogue, pause markers, built-in formant voices; Kokoro-82M natural voices on candle and their pinned package catalogue (feature `kokoro`). Same-layer edge to `tts-text` | +| `tts-text` | L2 | text front end for narration: normalizer (numbers, money, dates, acronyms, pauses) and English pronunciation (CMUdict + letter-to-sound) to Kokoro phonemes | | `codecs` | L2 | container + codec hub: MP4/MOV, MKV, MXF, Ogg and MPEG TS / PS / video elementary stream sources, GOP-aware seeking, decoder registry, audio decoding | | `interchange` | L2 | EDL, FCP7 XML, FCPXML, OTIO, AAF (on `cfb`) and OMF 2.0 import/export (no file I/O; the engine supplies rendered audio essence) ([README](../crates/interchange/README.md)) | | `render` | L3 | sequence evaluation, CPU compositor, video effects (`effects`, `vfx`; effects needing other frames or tracks read them through `vfx::FxEnv`), transitions, audio mix | @@ -455,6 +457,10 @@ stereo / mono track ──5.1 panner───┼─► 5.1 submix / 5.1 Mix ─ **Voice-over recording** (`engine::voiceover`, `audio.voiceover.*`). The record point R is the playhead, or the In point when In/Out are set (punch-in; punch-out at Out). Playback starts the pre-roll before it (C = max(0, R − pre-roll)) and the input is captured from C. When the UI's audio clock really starts, `audio.voiceover.sync` restarts the capture there. `audio.voiceover.stop` keeps the audio from R to min(stop, Out), writes a mono 32-bit float WAV ` .wav` (Scratch Disks ▸ Captured, else next to the project, else the data or temporary directory), imports it and overwrites it onto the record track at R as one undo step. The record track is the given one, else the record-armed one, else the first targeted one. Input goes through the `AudioInput` trait: cpal in the desktop app (`apps/filmcraft/src/audio_in.rs`, on its own thread), and `SyntheticInput` headless, which produces exactly the samples the timeline asks for so recordings land sample-accurately. Voice-Over Record Settings (Source, Input channel, Name, Countdown Sound Cues, Pre-/Post-roll) are preferences (`voiceOver`). The track header's microphone records or stops, and a right-click opens the settings dialog. Countdown beeps (1 kHz, 100 ms, each whole second of pre-roll and at R) are mixed into playback, an overlay counts down, and playback stops at Out + post-roll. +### 5.1.2a Text to Speech (narrations) + +**Narrations** (`engine::narration`, `tts.*`; voices in `filmcraft-tts`). The Text to Speech panel (Window ▸ Text to Speech) writes a script with pause markers (`[pause 1s]`), a voice, a vocal pitch and a pace; `tts.create` synthesizes it, writes `Narration .wav` (mono 32-bit float, 24 kHz, where voice-over recordings go), imports it and places it at the playhead on the first free targeted audio track, else the first free audio track, else a new one, as one undo step. The settings are kept per generated item in `Project::narrations` (schema v14). Selecting a narration clip loads it into the panel; `tts.edit` writes a new file (beside the previous one) and swaps the clip to it, keeping the clip's start, duration and speed: longer speech is kept whole in the file but plays only to the clip's end; shorter speech is padded with silence to the clip's length. The old item and narration stay, so undo is exact. `tts.preview` synthesizes without changing the project and the UI plays `Session::tts_preview` on the audio output. Built-in voices are an original formant synthesizer (robotic, no download). The natural voices (feature `neural-voices` of the desktop app, off by default like `whisper`) are Kokoro-82M on candle (`filmcraft-tts`) fed by `filmcraft-tts-text` (normalizer, CMUdict + letter-to-sound); their 336 MB package is downloaded once after the user confirms (`tts.downloadVoices`, a job). The UI synthesizes with `tts.render` (a job filling `Session::tts_cache`) and then runs create / edit / preview, which use the cache. + ### 5.1.3 Remix ```text diff --git a/docs/contributing.md b/docs/contributing.md index 570259e19..c38791fce 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -40,6 +40,7 @@ cargo xtask web --serve 8765 # the web app on http://12 |---|---| | `FILMCRAFT_CONTROL_PORT` | Same as `--control ` | | `FILMCRAFT_CPU_COMPOSITE=1` | Disables the GPU compositor | +| `FILMCRAFT_DEV_DESKTOP_ENTRY=1` | Linux, opt-in: a run from a source tree installs a user-level desktop entry (and the icons from `assets/app-icon`) so the taskbar shows the FilmCraft icon. Nothing is written without it; packaged runs never write one | | `FILMCRAFT_DATA_DIR` | Same as `--data-dir `: auto-save, crash recovery, settings and logs | | `RUST_LOG` | Log levels for standard error and the log file (see [Logs](#logs)) | diff --git a/docs/graphics.md b/docs/graphics.md index 4e1888fb3..433d491b2 100644 --- a/docs/graphics.md +++ b/docs/graphics.md @@ -89,7 +89,7 @@ Timecode and Clip Name effects also draw with the text engine. | `graphics.newShape` | (agents) | `shape` (rectangle/ellipse/polygon/path), `position` (the shape's centre), `size`, `points`, `clip` (add to this graphic), `seconds` (5), `track`, `time` | | `graphics.setTextType` | Text Properties ▸ Text Layer Type | `clip`, `layer`, `type` (`point` / `paragraph`); the text stays where it is | | `graphics.setText` | typing on the monitor | `clip`, `layer`, `text`, `merge` (coalesce one typing session into one undo step) | -| `graphics.set` | Properties panel | `clip`, `layer`, `props` {parameter id or camelCase alias: value; choices by index or name}, `time` | +| `graphics.set` | Properties panel | `clip`, `layer`, `props` {parameter id or camelCase alias: value; choices by index or name}, `time`, `merge` + `begin` (a drag is one undo step: send `merge: true` on every change and `begin: true` on the first change of each press) | | `graphics.selectLayer` | layer list / monitor click | `clip`, `layers` | | `graphics.deleteLayer`, `graphics.arrangeLayer` | layer list | `clip`, `layer`, `to` (front/back/forward/backward/index) | | `graphics.align` | Align and Transform | `align` (left/hcenter/right/top/vcenter/bottom), `to` (frame/group/selection), `layers` | diff --git a/docs/testing.md b/docs/testing.md index a453bed54..856dad73f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -31,6 +31,7 @@ FILMCRAFT_REQUIRE_ORACLES=1 cargo test --workspace # CI: missing ffmpeg fails | 5.1 / multichannel | `crates/audio-dsp/src/channels.rs`, `crates/render/src/mixer_tests.rs`, `crates/export/src/surround_tests.rs`, `crates/engine/src/mixer_tests.rs`, `crates/ui-egui/tests/audio51_ui.rs` | BS.775 downmix / upmix coefficients exact, mixdown types; 5.1 panner point gains equal-power (Σg² = 1 over a 21×21 grid × 3 centre settings), speaker positions exact, 5.1 through the default puck bit-exact identity; graph: stereo/mono tracks into a 5.1 Mix (L/R, C, rear right, room centre −6 dB each, LFE = mean), 5.1 track through a 5.1 Mix and folded into stereo, a 5.1 submix, `pan51` automation block-invariant, six-channel meters; export: 6-ch WAV (`WAVE_FORMAT_EXTENSIBLE`, mask 0x3F) and ProRes MOV (6 × 24-bit PCM) decoded by ffmpeg **sample-exact**, ffprobe `channels` 6 / `channel_layout` 5.1; AAC 5.1 in MP4: every decoded channel's own tone ≥ 30 dB above the others' (no swapped channels), stereo export of a 5.1 Mix = BS.775 within 2·10⁻⁶; `sequence.settings {mix}`, `file.newSequence {mix, trackType}`, panner lanes, Write pass records the puck; UI: puck drag (one undo step), Center / LFE, six meter bars | | Audio Clip Mixer automation, Track Mixer view | `crates/engine/src/mixer_tests.rs`, `crates/ui-egui/tests/audio51_ui.rs` | Touch: a 201-point fader ramp thins to ≤ 6 clip keyframes, values within 0.1 dB, live override while held, values outside the gesture unchanged, one undo step; Latch holds until stop; Write records volume and pan from the start; a gesture across two clips writes both; Read commits once without a pass; UI: Clip Mixer mode dropdown, a recorded fader drag writes clip keyframes; Show/Hide Tracks, Meter Input(s) Only, transport record button | | Voice-over | `crates/engine/src/voiceover_tests.rs`, `crates/ui-egui/tests/voiceover_ui.rs` | synthetic click train lands in `mix_sequence` exactly at R + k·period (pre-roll dropped, exact sample count), punch-in between In/Out (ramp source, every sample checked), one undo step + undo/redo, take numbering, record-arm/targeting track choice, sync/discard/zero-length/locked cases, cue times; UI: right-click Mic → Voice-Over Record Settings dialog (edits persist on OK, Cancel discards), the Mic button starts and stops a take on its track with pre-roll and overlay | +| Text to Speech | `crates/tts/src/**/*.rs` (Kokoro tests need `FILMCRAFT_KOKORO_DIR`), `crates/tts-text`, `crates/engine/src/narration_tests.rs`, `crates/ui-egui/tests/tts_ui.rs` | built-in voices: determinism, finite samples, exact-silence pause markers, pace scales length, measured pitch per voice and per semitone, hostile scripts; engine: narration placed at the playhead with clip length = speech (sample-exact), never overwrites (busy tracks → next free track → new track), occupied/locked/unknown track refused with no change, edit keeps start/length/speed (longer speech: whole speech in the file; shorter: file padded with exact zeros), edits write beside the previous file, undo/redo restore the project exactly, enabled only with a narration clip (or `clip`), 16 hostile parameter sets refused without change, preview changes nothing, save/reload keeps narrations, the narration is audible in `mix_sequence`; UI: type + Add pause + Add to timeline, the new clip is selected and loaded, clicking the clip on the timeline loads its script, Save edits it in place as one undo step, Advanced twirl, panel opens beside Properties; natural voices: 50-sentence Whisper-base WER 2.1–3.6 % for all 9 voices (crate README), loaders fuzzed with hostile files, `tts.render` cache, download refused without the feature | | Remix | `crates/audio-dsp/src/remix.rs`, `crates/render/src/remix.rs`, `crates/engine/src/remix_tests.rs`, `crates/ui-egui/tests/remix_ui.rs` | generated rhythmic music (Rust-synthesised drums, bass, chord sections at 90–128 BPM, 22.05/44.1/48 kHz): tempo within 0.3 BPM, every beat within 10 ms (measured ≤ 8.9 ms); remix targets from 0.4× to 2.2× the source within one beat (measured ≤ 12 ms at default sliders), every cut on a detected beat, intro and outro kept, pieces ≥ 4 beats, deterministic; silence, tone and noise refused; rendering independent of request cuts, source-exact outside the 20 ms equal-power crossfades, no clicks; the engine's mix equals the source pieces (< 1e-5); undo, redo and revert exact; overlaps refused; Remix Properties dialog and Remix tool drag driven by automation id | | Oracle | `crates/*/tests/*oracle*.rs`, `conformance.rs` | compare with ffmpeg/ffprobe (§2) | | Robustness / fuzz | `*/tests/robustness.rs`, `*/tests/fuzz.rs` | seeded mutation and truncation of real and synthetic files; nothing may panic | diff --git a/xtask/src/main.rs b/xtask/src/main.rs index fb5419463..631678d84 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -60,6 +60,8 @@ const LAYERS: &[(&str, u8)] = &[ ("audio", 2), ("captions", 2), ("speech", 2), + ("tts", 2), + ("tts-text", 2), ("interchange", 2), ("render", 3), ("gpu", 3), @@ -93,6 +95,7 @@ const SAME_LAYER: &[(&str, &str)] = &[ ("scopes", "gpu"), ("gpu", "render"), ("cli", "filmcraft"), + ("tts", "tts-text"), ]; /// Crates that must not appear below L5 (UI toolkits, windowing, OS audio/menus).