//! Text-based editing commands (`transcript.*`): the Text panel ▸ Transcript tab. //! //! Transcripts belong to media items (`Project::transcripts`, media time); the sequence transcript //! is derived from them ([`filmcraft_edit::transcript::sequence_words`]). Words of the sequence //! transcript are addressed by index (`from`, `to`, inclusive), as `transcript.inspect` lists them. //! //! Speech recognition goes through a [`Transcriber`]: [`Session::transcriber`] when a host or a //! test installed one, else the Whisper model named by `model` from `/models` (needs the //! engine feature `whisper`; without it `transcript.generate` fails with a clear error, and agents //! can still bring their own transcript with `transcript.set`). use std::sync::Arc; use serde_json::{Value, json}; use filmcraft_edit as edit; use filmcraft_edit::transcript::{self as tx, CaptionRules, SeqWord}; use filmcraft_project::{CaptionFormat, CaptionTrack, ItemId, ItemKind, TrackId, Transcript}; use filmcraft_speech::{Options, SpeechError, Transcriber}; use filmcraft_time::{TICKS_PER_SECOND, Tick, TimeRange}; use crate::commands::{CommandSpec, always, bad, bool_p, f64_p, has_seq, str_p, u64_p}; use crate::{EngineError, Result, Session}; type Run = fn(&mut Session, &Value) -> Result; type Enabled = fn(&Session) -> std::result::Result<(), String>; fn spec(id: &'static str, label: &'static str, menu: &'static [&'static str], params: &'static str, enabled: Enabled, run: Run, journal: bool) -> CommandSpec { CommandSpec { id, label, menu, shortcut: None, params, enabled, run, journal } } /// Where downloaded speech models live (`/models`). pub fn models_dir() -> Option { crate::autosave::default_data_dir().map(|d| d.join("models")) } /// Whether this build can transcribe with Whisper (feature `whisper`). pub fn speech_available() -> bool { filmcraft_speech::available() } /// The words of the active sequence's transcript. pub fn sequence_words(s: &Session) -> Vec { match s.active_sequence() { Some(q) => tx::sequence_words(q, &s.project.transcripts), None => Vec::new(), } } fn has_transcript(s: &Session) -> std::result::Result<(), String> { has_seq(s)?; if sequence_words(s).is_empty() { Err("the sequence has no transcript (Transcribe first)".into()) } else { Ok(()) } } fn has_transcripts(s: &Session) -> std::result::Result<(), String> { if s.project.transcripts.is_empty() { Err("there are no transcripts".into()) } else { Ok(()) } } /// The media item behind a project item (subclips resolve to their parent). fn media_item(s: &Session, item: ItemId) -> Option { match &s.project.item(item)?.kind { ItemKind::Media(_) => Some(item), ItemKind::Subclip { parent, .. } => media_item(s, *parent), _ => None, } } fn ids_p(p: &Value, k: &str) -> Option> { p.get(k).and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_u64).map(ItemId).collect()) } /// Items to transcribe: `items` / `item`, else the Project panel selection, else the media of the /// active sequence's enabled audio clips. Subclips resolve to their media; duplicates are removed. fn targets(s: &Session, p: &Value) -> Vec { let mut raw = ids_p(p, "items").or_else(|| u64_p(p, "item").map(|i| vec![ItemId(i)])).unwrap_or_default(); if raw.is_empty() { raw = s.state.project_selection.clone(); } if raw.is_empty() && let Some(q) = s.active_sequence() { raw = q.audio_tracks.iter().flat_map(|t| t.items.iter()).filter(|it| it.enabled).map(|it| it.item).collect(); } let mut out = Vec::new(); for i in raw { if let Some(m) = media_item(s, i) && !out.contains(&m) { out.push(m); } } out } /// Mono 16 kHz audio of a media item (None: no audio). fn item_audio(s: &Session, item: ItemId) -> Option> { let dur = match &s.project.item(item)?.kind { ItemKind::Media(m) => m.duration(), _ => return None, }; let src = s.source(item)?; if !src.info().has_audio() { return None; } let sr = filmcraft_speech::SAMPLE_RATE; let len = dur.to_units_floor(sr as i64).max(0) as usize; let buf = src.audio(0, len, sr).ok()?; Some(filmcraft_speech::downmix(&buf.channels)) } fn speech_err(e: SpeechError) -> EngineError { EngineError::Other(e.to_string()) } /// The transcriber to use: the installed one, else the named catalogue model. fn transcriber(s: &Session, p: &Value) -> Result> { if let Some(t) = &s.transcriber { return Ok(t.clone()); } // Settings ▸ Media Analysis & Transcription ▸ Speech model let model = str_p(p, "model").unwrap_or(&s.prefs.media_analysis.whisper_model); if filmcraft_speech::models::find(model).is_none() { return Err(speech_err(SpeechError::UnknownModel(model.into()))); } if !filmcraft_speech::available() { return Err(EngineError::Other( "speech-to-text is not available in this build (built without the `whisper` feature); import a transcript with transcript.set instead".into(), )); } let dir = models_dir().ok_or_else(|| EngineError::Other("no data directory for speech models".into()))?; filmcraft_speech::load(&dir, model).map_err(speech_err) } fn generate(s: &mut Session, p: &Value) -> Result { let items = targets(s, p); if items.is_empty() { return Err(bad("transcript.generate", "nothing to transcribe (pass `items`, select clips, or open a sequence with audio)")); } let t = transcriber(s, p)?; // Settings ▸ Media Analysis & Transcription: language (or auto-detect) and speaker labelling let ma = &s.prefs.media_analysis; let default_language = if ma.language_auto_detect { None } else { Some(ma.default_language.clone()) }; let opts = Options { language: match str_p(p, "language") { Some(l) => Some(l).filter(|l| !l.is_empty() && *l != "auto").map(str::to_string), None => default_language, }, diarize: bool_p(p, "diarize").unwrap_or(ma.speaker_labeling != "off"), max_speakers: u64_p(p, "maxSpeakers").map(|n| n.clamp(1, 32) as usize).unwrap_or(Options::default().max_speakers), }; let mut done: Vec<(ItemId, Transcript)> = Vec::new(); let mut skipped = Vec::new(); for item in items { let Some(audio) = item_audio(s, item) else { skipped.push(item.0); continue; }; let mut tr = t.transcribe(&audio, &opts, &mut |_, _| true).map_err(speech_err)?; tr.normalize(); done.push((item, tr)); } if done.is_empty() { return Err(EngineError::Other("none of the clips has audio to transcribe".into())); } let report: Vec = done .iter() .map(|(i, t)| json!({"item": i.0, "words": t.words.len(), "speakers": t.speakers.len(), "language": t.language, "source": t.source})) .collect(); s.edit("Transcribe", move |pr, _| { for (i, t) in done { pr.transcripts.insert(i, Arc::new(t)); } Ok(()) })?; Ok(json!({"items": report, "skipped": skipped})) } fn set(s: &mut Session, p: &Value) -> Result { let item = u64_p(p, "item").map(ItemId).ok_or_else(|| bad("transcript.set", "`item` is required"))?; let item = media_item(s, item).ok_or_else(|| bad("transcript.set", "no such media item"))?; let v = p.get("transcript").cloned().ok_or_else(|| bad("transcript.set", "`transcript` is required"))?; let mut t: Transcript = serde_json::from_value(v).map_err(|e| bad("transcript.set", e.to_string()))?; if t.source.is_empty() { t.source = "imported".into(); } t.normalize(); t.check().map_err(|e| bad("transcript.set", e))?; let n = t.words.len(); s.edit("Set Transcript", move |pr, _| { pr.transcripts.insert(item, Arc::new(t)); Ok(()) })?; Ok(json!({"item": item.0, "words": n})) } fn delete(s: &mut Session, p: &Value) -> Result { let items: Vec = match ids_p(p, "items").or_else(|| u64_p(p, "item").map(|i| vec![ItemId(i)])) { Some(v) => v.into_iter().filter_map(|i| media_item(s, i)).collect(), None => s.project.transcripts.keys().copied().collect(), }; let n = items.iter().filter(|i| s.project.transcripts.contains_key(i)).count(); if n == 0 { return Err(EngineError::Other("no transcript to delete".into())); } s.edit("Delete Transcript", move |pr, _| { for i in items { pr.transcripts.remove(&i); } Ok(()) })?; Ok(json!({"deleted": n})) } fn word_json(i: usize, w: &SeqWord) -> Value { json!({"i": i, "text": w.text, "start": w.start.0, "end": w.end.0, "speaker": w.speaker, "clip": w.clip.0, "item": w.item.0, "confidence": w.confidence}) } fn inspect(s: &mut Session, p: &Value) -> Result { let words = sequence_words(s); let gap = Tick::from_seconds_f64(f64_p(p, "paragraphGapSeconds").unwrap_or(1.5)); let paras: Vec = tx::paragraphs(&words, gap) .into_iter() .map(|r| { let text = words[r.clone()].iter().map(|w| w.text.as_str()).collect::>().join(" "); json!({"from": r.start, "to": r.end - 1, "speaker": words[r.start].speaker, "start": words[r.start].start.0, "end": words[r.end - 1].end.0, "text": text}) }) .collect(); let speakers: Vec = { let mut v: Vec = Vec::new(); for w in &words { if let Some(n) = &w.speaker && !v.contains(n) { v.push(n.clone()); } } v }; let current = tx::word_at(&words, s.playhead()); Ok(json!({ "words": words.iter().enumerate().map(|(i, w)| word_json(i, w)).collect::>(), "paragraphs": paras, "speakers": speakers, "current": current, "items": s.project.transcripts.iter().map(|(i, t)| json!({"item": i.0, "words": t.words.len(), "language": t.language, "source": t.source, "speakers": t.speakers.iter().map(|k| &k.name).collect::>()})).collect::>(), })) } fn search(s: &mut Session, p: &Value) -> Result { let q = str_p(p, "query").ok_or_else(|| bad("transcript.search", "`query` is required"))?; let words = sequence_words(s); let hits: Vec = tx::search(&words, q) .into_iter() .map(|r| json!({"from": r.start, "to": r.end - 1, "start": words[r.start].start.0, "end": words[r.end - 1].end.0})) .collect(); Ok(json!({"matches": hits})) } /// Timeline range of the words `from..=to` (frame-snapped outward). fn range_p(s: &Session, p: &Value, cmd: &str) -> Result { let words = sequence_words(s); let from = u64_p(p, "from").ok_or_else(|| bad(cmd, "`from` (word index) is required"))? as usize; let to = u64_p(p, "to").map(|n| n as usize).unwrap_or(from); tx::word_range(&words, from, to, s.sequence_rate()).ok_or_else(|| bad(cmd, format!("word index out of range (the transcript has {} words)", words.len()))) } fn range_json(r: TimeRange) -> Value { json!({"start": r.start.0, "end": r.end().0}) } fn select(s: &mut Session, p: &Value) -> Result { let r = range_p(s, p, "transcript.select")?; let fd = s.sequence_rate().frame_duration(); s.edit_sequence("Mark Transcript Selection", |q, _, _| { q.mark_in = Some(r.start); q.mark_out = Some(r.end() - fd); Ok(()) })?; s.set_playhead(r.start); Ok(range_json(r)) } fn extract_or_lift(s: &mut Session, p: &Value, extract: bool) -> Result { let cmd = if extract { "transcript.extract" } else { "transcript.lift" }; let r = range_p(s, p, cmd)?; let tg = s.targeting().targeted; s.edit_sequence(if extract { "Extract Text" } else { "Lift Text" }, |q, ctx, _| { if extract { edit::extract(q, &tg, r, ctx); } else { edit::lift(q, &tg, r, ctx); } q.mark_in = None; q.mark_out = None; Ok(()) })?; s.set_playhead(r.start); Ok(range_json(r)) } fn rename_speaker(s: &mut Session, p: &Value) -> Result { let name = str_p(p, "name").map(str::trim).filter(|n| !n.is_empty()).ok_or_else(|| bad("transcript.renameSpeaker", "`name` is required"))?.to_string(); let item = u64_p(p, "item").map(ItemId).and_then(|i| media_item(s, i)); // `speaker`: the current name (every transcript), or an index (needs `item`) let (old_name, index) = match p.get("speaker") { Some(Value::String(n)) => (Some(n.clone()), None), Some(v) if v.is_u64() => (None, v.as_u64().map(|n| n as usize)), _ => return Err(bad("transcript.renameSpeaker", "`speaker` (name, or index with `item`) is required")), }; if index.is_some() && item.is_none() { return Err(bad("transcript.renameSpeaker", "a speaker index needs `item`")); } let mut n = 0; let mut next = s.project.transcripts.clone(); for (i, t) in next.iter_mut() { if item.is_some_and(|x| x != *i) { continue; } let tt = Arc::make_mut(t); for (k, sp) in tt.speakers.iter_mut().enumerate() { if old_name.as_ref().is_some_and(|o| *o == sp.name) || index == Some(k) { sp.name = name.clone(); n += 1; } } } if n == 0 { return Err(EngineError::Other("no such speaker".into())); } s.edit("Rename Speaker", move |pr, _| { pr.transcripts = next; Ok(()) })?; Ok(json!({"renamed": n})) } fn remove_ranges(s: &mut Session, label: &str, ranges: Vec) -> Result { let n = ranges.len(); if n == 0 { return Ok(json!({"removed": 0, "ticks": 0})); } let total = s.edit_sequence(label, |q, ctx, _| Ok(tx::ripple_delete_ranges(q, ranges, ctx)))?; Ok(json!({"removed": n, "ticks": total.0, "seconds": total.0 as f64 / TICKS_PER_SECOND as f64})) } fn remove_pauses(s: &mut Session, p: &Value) -> Result { let words = sequence_words(s); let min = Tick::from_seconds_f64(f64_p(p, "minSeconds").unwrap_or(1.0)); let keep = Tick::from_seconds_f64(f64_p(p, "keepSeconds").unwrap_or(0.15)); let ranges = tx::find_pauses(&words, min, keep, s.sequence_rate()); remove_ranges(s, "Remove Pauses", ranges) } fn remove_fillers(s: &mut Session, p: &Value) -> Result { let words = sequence_words(s); let fillers: Vec = match p.get("fillers").and_then(Value::as_array) { Some(a) => a.iter().filter_map(Value::as_str).map(str::to_string).collect(), None => tx::DEFAULT_FILLERS.iter().map(|f| f.to_string()).collect(), }; let hits = tx::find_fillers(&words, &fillers); let ranges = tx::filler_ranges(&words, &hits, s.sequence_rate()); remove_ranges(s, "Remove Filler Words", ranges) } fn create_captions(s: &mut Session, p: &Value) -> Result { let words = sequence_words(s); let d = CaptionRules::default(); let rules = CaptionRules { max_chars: u64_p(p, "maxChars").map(|n| n as usize).unwrap_or(d.max_chars), lines: u64_p(p, "lines").map(|n| n as usize).unwrap_or(d.lines), min_duration: f64_p(p, "minSeconds").map(Tick::from_seconds_f64).unwrap_or(d.min_duration), max_duration: f64_p(p, "maxSeconds").map(Tick::from_seconds_f64).unwrap_or(d.max_duration), gap_frames: p.get("gapFrames").and_then(Value::as_i64).unwrap_or(d.gap_frames), break_pause: d.break_pause, }; let blocks = tx::caption_blocks(&words, &rules, s.sequence_rate()); let format = str_p(p, "format").and_then(CaptionFormat::from_name).unwrap_or_default(); let name = str_p(p, "name").unwrap_or("Transcript").to_string(); let n = blocks.len(); let tid = s.edit_sequence("Create Captions", |q, ctx, st| { let tid = TrackId(ctx.alloc()); let mut t = CaptionTrack::new(tid, name, format); t.captions = tx::blocks_to_captions(&blocks, ctx); q.caption_tracks.insert(0, t); st.caption_selection.clear(); Ok(tid) })?; Ok(json!({"track": tid.0, "captions": n})) } fn models(_: &mut Session, _: &Value) -> Result { let dir = models_dir(); Ok(json!({ "available": filmcraft_speech::available(), "default": filmcraft_speech::models::DEFAULT_MODEL, "dir": dir.as_ref().map(|d| d.to_string_lossy().to_string()), "models": filmcraft_speech::models::catalogue().iter().map(|m| json!({ "id": m.id, "name": m.name, "multilingual": m.multilingual, "description": m.description, "license": m.license, "source": m.source, "size": m.size(), "installed": dir.as_ref().is_some_and(|d| filmcraft_speech::models::installed(d, m)), })).collect::>(), })) } /// Download a catalogue model into `/models` (feature `speech-download`). Hosts show /// the size, source and licence (`transcript.models`) and ask before running this. fn download_model(_: &mut Session, p: &Value) -> Result { let id = str_p(p, "model").unwrap_or(filmcraft_speech::models::DEFAULT_MODEL); let m = filmcraft_speech::models::find(id).ok_or_else(|| speech_err(SpeechError::UnknownModel(id.into())))?; let dir = models_dir().ok_or_else(|| EngineError::Other("no data directory for speech models".into()))?; #[cfg(feature = "speech-download")] { filmcraft_speech::models::download(&dir, m, &mut |_, _, _| true).map_err(speech_err)?; Ok(json!({"model": m.id, "dir": filmcraft_speech::models::model_dir(&dir, m).to_string_lossy()})) } #[cfg(not(feature = "speech-download"))] { let _ = (m, dir); Err(EngineError::Other("model downloads are not available in this build (built without the `speech-download` feature)".into())) } } pub fn commands() -> Vec { vec![ spec( "transcript.generate", "Transcribe…", &["Sequence", "Transcript"], r#"{"items":[id]?,"model":"whisper-base"?,"language":"en|auto"?,"diarize":bool?,"maxSpeakers":n?}"#, always, generate, true, ), spec( "transcript.set", "Import Transcript", &[], r#"{"item":id,"transcript":{"language":str,"speakers":[{"name":str}],"words":[{"text":str,"start":tick,"end":tick,"speaker":n?}]}}"#, always, set, true, ), spec("transcript.delete", "Delete Transcript", &["Sequence", "Transcript"], r#"{"items":[id]?}"#, has_transcripts, delete, true), spec("transcript.inspect", "Inspect Transcript", &[], r#"{"paragraphGapSeconds":f?}"#, always, inspect, false), spec("transcript.search", "Search Transcript", &[], r#"{"query":str}"#, always, search, false), spec("transcript.models", "List Speech Models", &[], "{}", always, models, false), spec("transcript.downloadModel", "Download Speech Model", &[], r#"{"model":"whisper-base"?}"#, always, download_model, true), spec("transcript.select", "Mark Selected Text", &[], r#"{"from":word,"to":word?}"#, has_transcript, select, true), spec("transcript.extract", "Extract Selected Text", &[], r#"{"from":word,"to":word?}"#, has_transcript, |s, p| extract_or_lift(s, p, true), true), spec("transcript.lift", "Lift Selected Text", &[], r#"{"from":word,"to":word?}"#, has_transcript, |s, p| extract_or_lift(s, p, false), true), spec( "transcript.renameSpeaker", "Rename Speaker…", &[], r#"{"speaker":"Speaker 1"|index,"name":str,"item":id?}"#, has_transcripts, rename_speaker, true, ), spec( "transcript.removePauses", "Remove Pauses", &["Sequence", "Transcript"], r#"{"minSeconds":f?,"keepSeconds":f?}"#, has_transcript, remove_pauses, true, ), spec("transcript.removeFillers", "Remove Filler Words", &["Sequence", "Transcript"], r#"{"fillers":[str]?}"#, has_transcript, remove_fillers, true), spec( "transcript.createCaptions", "Create Captions from Transcript…", &["Sequence", "Transcript"], r#"{"maxChars":n?,"lines":1|2?,"minSeconds":f?,"maxSeconds":f?,"gapFrames":n?,"format":str?,"name":str?}"#, has_transcript, create_captions, true, ), ] }