diff --git a/apps/effectcraft-web/src/app.rs b/apps/effectcraft-web/src/app.rs index 1a695240..68fd1a40 100644 --- a/apps/effectcraft-web/src/app.rs +++ b/apps/effectcraft-web/src/app.rs @@ -257,6 +257,7 @@ pub async fn start(canvas_id: String) -> Result<(), JsValue> { if n > 0 { app.frames.set_remote(Some(Arc::new(frames::WebFrames::start(n, !query_flag("nogpuworkers"))))); } + app.frames.set_virtual_media(files::STORE.media_epoch(), Arc::new(|path| files::STORE.file_identity(path))); let (tx, rx) = std::sync::mpsc::channel(); api::set_sender(tx); app = app.with_control(rx); diff --git a/apps/effectcraft-web/src/frames.rs b/apps/effectcraft-web/src/frames.rs index 523fd03c..c7f35052 100644 --- a/apps/effectcraft-web/src/frames.rs +++ b/apps/effectcraft-web/src/frames.rs @@ -416,6 +416,7 @@ fn post_reply(r: &FrameReply, px: Option<&[u8]>) { fn new_server() -> FrameServer { let mut s = FrameServer::new(crate::worker::pool(), Some(Arc::new(effectcraft_expr::Expressions))); s.set_layer_store(Some(Arc::new(effectcraft_engine::render::PrefetchStore::default()))); + s.set_virtual_id(Some(Arc::new(|path| crate::files::STORE.file_identity(path)))); s } diff --git a/apps/effectcraft-web/src/store.rs b/apps/effectcraft-web/src/store.rs index b3a858de..a44a2eac 100644 --- a/apps/effectcraft-web/src/store.rs +++ b/apps/effectcraft-web/src/store.rs @@ -20,6 +20,7 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, OnceLock}; use effectcraft_engine::config::{ConfigStore, FileOps}; @@ -45,6 +46,8 @@ pub struct Entry { /// Which write this is: new with every write of the key, so a file replaced by another of /// the same size still reads as changed (workers are sent it again, #218). pub version: u64, + /// FNV-1a of `data`, filled the first time a disk-cache key asks for it. + hash: Option, } /// A queued write for the browser storage: `data` = `None` deletes the key. @@ -69,13 +72,13 @@ impl Mirror { /// An entry read from the browser's storage at startup (nothing to flush). pub fn load(&mut self, key: &str, data: Arc<[u8]>, modified: f64) { let version = self.next_version(); - self.entries.insert(key.to_string(), Entry { data, modified, persist: true, version }); + self.entries.insert(key.to_string(), Entry { data, modified, persist: true, version, hash: None }); } pub fn put(&mut self, key: &str, data: Arc<[u8]>, modified: f64, persist: bool) { let was_persisted = self.entries.get(key).is_some_and(|e| e.persist); let version = self.next_version(); - self.entries.insert(key.to_string(), Entry { data, modified, persist, version }); + self.entries.insert(key.to_string(), Entry { data, modified, persist, version, hash: None }); if persist || was_persisted { self.touch(key); } @@ -149,14 +152,31 @@ pub fn key_path(key: &str) -> Option<&str> { key.strip_prefix(FILES).filter(|p| p.starts_with('/')) } +/// FNV-1a 64. Computed when a disk-cache key first needs the file, not on every write. +fn content_hash(data: &[u8]) -> u64 { + let mut h: u64 = 0xcbf2_9ce4_8422_2325; + for &b in data { + h ^= u64::from(b); + h = h.wrapping_mul(0x0000_0100_0000_01b3); + } + h +} + /// The persistent store: a [`Mirror`] and a hook that wakes the flusher after changes. -#[derive(Default)] pub struct Store { pub mirror: Mutex, /// Called after every change that needs flushing (the web host schedules a flush). pub on_change: OnceLock, /// Clock in milliseconds since the Unix epoch (tests pin it). pub clock: OnceLock f64>, + /// Bumped when a virtual file is written. Preview cache lookups key on it; they do not hash. + media_epoch: Arc, +} + +impl Default for Store { + fn default() -> Self { + Self { mirror: Mutex::default(), on_change: OnceLock::new(), clock: OnceLock::new(), media_epoch: Arc::new(AtomicU64::new(0)) } + } } impl Store { @@ -179,12 +199,40 @@ impl Store { /// Store a file under `path` (persisted unless `persist` is false: render outputs). pub fn put_file(&self, path: &str, data: Arc<[u8]>, persist: bool) { let now = self.now(); - self.lock().put(&file_key(path), data, now, persist); + let key = file_key(path); + let replaced = self.lock().get(&key).is_none_or(|e| !Arc::ptr_eq(&e.data, &data)); + self.lock().put(&key, data, now, persist); + if replaced { + self.media_epoch.fetch_add(1, Ordering::Relaxed); + } if persist { self.changed(); } } + /// The generation preview cache lookups key on. It changes when [`Self::put_file`] writes + /// a different blob, and it is not a content hash. + pub fn media_epoch(&self) -> Arc { + self.media_epoch.clone() + } + + /// Size and content hash of the file under `path`. The hash is stored on the entry so a + /// later key does not walk the bytes again. + pub fn file_identity(&self, path: &str) -> Option<(u64, u64)> { + let key = file_key(path); + let mut mirror = self.lock(); + let entry = mirror.entries.get(&key)?; + if let Some(hash) = entry.hash { + return Some((entry.data.len() as u64, hash)); + } + let hash = content_hash(&entry.data); + let len = entry.data.len() as u64; + if let Some(entry) = mirror.entries.get_mut(&key) { + entry.hash = Some(hash); + } + Some((len, hash)) + } + pub fn file(&self, path: &str) -> Option> { self.lock().get(&file_key(path)).map(|e| e.data.clone()) } @@ -446,6 +494,11 @@ mod tests { assert_ne!(a, b); assert_eq!(&data[..], b"blue"); assert_eq!(s.file_versioned("/collision.png").map(|f| f.1), Some(b), "reading doesn't change it"); + let epoch = s.media_epoch().load(Ordering::Relaxed); + let id = s.file_identity("/collision.png").expect("identity"); + assert_eq!(id, (4, content_hash(b"blue"))); + assert_eq!(s.file_identity("/collision.png"), Some(id), "the hash is kept on the entry"); + assert!(epoch >= 2, "each new blob bumps the generation the preview keys on"); } /// A write that failed is flushed again with the key's latest state (#215). diff --git a/crates/engine/src/remote.rs b/crates/engine/src/remote.rs index 207367b0..7d592abe 100644 --- a/crates/engine/src/remote.rs +++ b/crates/engine/src/remote.rs @@ -263,6 +263,8 @@ pub struct FrameServer { /// The disk cache's layer buffers (see the module docs) and the project's salt. layer_store: Option>, salt: u64, + /// Size and content hash for footage paths with no native metadata (the browser file table). + virtual_id: Option Option<(u64, u64)> + Send + Sync>>, } /// A frame whose readbacks are in flight. @@ -294,9 +296,16 @@ impl FrameServer { pending: None, layer_store: None, salt: 0, + virtual_id: None, } } + /// Identity of virtual footage files, used when native metadata cannot describe the path. + /// Called from [`FrameServer::load`], not from the UI thread's cache lookup. + pub fn set_virtual_id(&mut self, id: Option Option<(u64, u64)> + Send + Sync>>) { + self.virtual_id = id; + } + /// The store behind the layer cache when a request asks for `layers` (the browser: the /// disk cache's layer buffers, fetched before the request). pub fn set_layer_store(&mut self, store: Option>) { @@ -338,7 +347,7 @@ impl FrameServer { fn load(&mut self, revision: u64) -> Result<(), String> { let p = Project::from_json(&self.doc.to_string()).map_err(|e| e.to_string())?; - self.salt = effectcraft_render::disk_cache::footage_salt(&p); + self.salt = effectcraft_render::disk_cache::footage_salt_with(&p, self.virtual_id.as_deref()); self.cache.set_store_salt(self.salt); self.project = Some(Arc::new(p)); self.revision = Some(revision); diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index 4140b792..f867a614 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -779,8 +779,12 @@ fn hash_debug(h: &mut Hash128, v: &impl std::fmt::Debug) { h.write(&[0xff]); } -/// A file's size and modification time (so edited footage gets new keys). -fn file_stamp(h: &mut Hash128, path: &str) { +/// Size and content hash of a path that has no native metadata (the browser file table). +pub type VirtualFileId<'a> = &'a (dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync); + +/// A file's size and modification time (so edited footage gets new keys). When the path has no +/// native metadata, `virtual_id` supplies a durable size and content hash instead. +fn file_stamp(h: &mut Hash128, path: &str, virtual_id: Option>) { if path.is_empty() { return; } @@ -791,26 +795,36 @@ fn file_stamp(h: &mut Hash128, path: &str) { { h.write_u64(d.as_nanos() as u64); } + return; + } + if let Some((len, hash)) = virtual_id.and_then(|id| id(path)) { + h.write_u64(len); + h.write_u64(hash); } } -fn footage_stamp(h: &mut Hash128, f: &effectcraft_project::Footage) { - file_stamp(h, &f.path); +fn footage_stamp(h: &mut Hash128, f: &effectcraft_project::Footage, virtual_id: Option>) { + file_stamp(h, &f.path, virtual_id); for p in &f.sequence { - file_stamp(h, p); + file_stamp(h, p, virtual_id); } } /// Salt for layer cache keys on disk: the renderer version and every footage file's identity /// (layer keys name footage items, not file contents). pub fn footage_salt(project: &effectcraft_project::Project) -> u64 { + footage_salt_with(project, None) +} + +/// [`footage_salt`] with [`VirtualFileId`] for paths `std::fs::metadata` cannot describe. +pub fn footage_salt_with(project: &effectcraft_project::Project, virtual_id: Option>) -> u64 { let mut h = Hash128::default(); h.write(RENDER_VERSION.as_bytes()); for (id, it) in &project.items { if let effectcraft_project::ItemKind::Footage(f) = &it.kind { h.write_u64(id.0); hash_debug(&mut h, f); - footage_stamp(&mut h, f); + footage_stamp(&mut h, f, virtual_id); } } let v = h.finish(); @@ -822,6 +836,11 @@ pub fn footage_salt(project: &effectcraft_project::Project) -> u64 { /// files' size and time). With expressions in play (which can read any comp) the whole project /// counts. Switches that don't change pixels (Audio, Lock, Shy, Hide Shy Layers) don't count. pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId) -> u128 { + comp_content_key_with(project, comp, None) +} + +/// [`comp_content_key`] with [`VirtualFileId`] for paths `std::fs::metadata` cannot describe. +pub fn comp_content_key_with(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId, virtual_id: Option>) -> u128 { use effectcraft_project::{ItemKind, LayerSource}; let mut h = Hash128::default(); h.write(RENDER_VERSION.as_bytes()); @@ -855,11 +874,11 @@ pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraf k => hash_debug(&mut h, k), } if let ItemKind::Footage(f) = &it.kind { - footage_stamp(&mut h, f); + footage_stamp(&mut h, f, virtual_id); } hash_debug(&mut h, &it.proxy); if let Some(px) = &it.proxy { - footage_stamp(&mut h, &px.footage); + footage_stamp(&mut h, &px.footage, virtual_id); } } h.write_u64(comp.0); @@ -1082,4 +1101,29 @@ mod tests { assert_eq!(b.get_layer(k).unwrap().img, buf(k as u32, 16, 16).img); } } + + /// A path with no native metadata takes its key from the caller's size and content hash (#1253). + #[test] + fn virtual_file_bytes_change_content_keys() { + use effectcraft_project::build; + use effectcraft_project::{Comp, Footage, ItemKind, LayerSource, Project}; + use effectcraft_time::{FrameRate, Tick}; + + let mut p = Project::default(); + let f = Footage { path: "/virtual.mov".into(), width: 16, height: 9, duration: Tick::from_seconds_f64(1.0), has_video: true, ..Default::default() }; + let fid = p.add_item("virtual.mov", effectcraft_color::Label::None, None, ItemKind::Footage(f)); + let mut comp = Comp::new(16, 9, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); + comp.layers = vec![build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (16, 9), None)]; + let cid = p.add_item("C", effectcraft_color::Label::None, None, ItemKind::Comp(comp.into())); + + let id = |mark: u64| move |path: &str| (path == "/virtual.mov").then_some((8, mark)); + let (one, two) = (id(1), id(2)); + let a = footage_salt_with(&p, Some(&one)); + let b = footage_salt_with(&p, Some(&two)); + let ka = comp_content_key_with(&p, cid, Some(&one)); + let kb = comp_content_key_with(&p, cid, Some(&two)); + assert_ne!(a, b, "replaced virtual media must not reuse the salt"); + assert_ne!(ka, kb, "replaced virtual media must not reuse the frame key"); + assert_eq!(comp_content_key(&p, cid), comp_content_key_with(&p, cid, None)); + } } diff --git a/crates/ui-egui/src/frames.rs b/crates/ui-egui/src/frames.rs index 059bc3a0..a0a89345 100644 --- a/crates/ui-egui/src/frames.rs +++ b/crates/ui-egui/src/frames.rs @@ -24,7 +24,7 @@ //! them (Info panel, eyedroppers, histograms). use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; use effectcraft_engine::project::{Comp, ItemId, ItemKind, LayerSource, Node, Project, PropGroup}; @@ -258,8 +258,10 @@ pub struct RenderSource { /// ([`Frames::identity`]). type Identity = (u64, Option>); -/// Content keys of (project revision, comp), computed once per revision. -type ContentKeys = Arc>>; +/// Content keys of (project revision, comp, virtual-file generation). The generation is one +/// counter bumped when a browser file is replaced, so a lookup stays a map read. Hashing the +/// new bytes happens only when that generation misses. +type ContentKeys = Arc>>; #[cfg(not(target_arch = "wasm32"))] #[derive(Default)] @@ -268,16 +270,16 @@ struct ContentKeyJobs { pending: VecDeque<(Arc, u64, u64)>, } -fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64) -> u128 { - if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp)).copied()) { +fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64, epoch: u64, virtual_id: Option>) -> u128 { + if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp, epoch)).copied()) { return k; } - let k = disk_cache::comp_content_key(project, ItemId(comp)); + let k = disk_cache::comp_content_key_with(project, ItemId(comp), virtual_id); if let Ok(mut m) = keys.lock() { if m.len() > 256 { m.clear(); } - m.insert((revision, comp), k); + m.insert((revision, comp, epoch), k); } k } @@ -363,8 +365,15 @@ pub fn opts_hash(o: &RenderOpts) -> u64 { } /// Disk key of a viewer frame (render options included: region of interest, draft…). -fn frame_disk_key(keys: &ContentKeys, project: &Project, key: &FrameKey, opts_hash: u64) -> u128 { - let content = content_key(keys, project, key.revision, key.comp); +fn frame_disk_key( + keys: &ContentKeys, + project: &Project, + key: &FrameKey, + opts_hash: u64, + epoch: u64, + virtual_id: Option>, +) -> u128 { + let content = content_key(keys, project, key.revision, key.comp, epoch, virtual_id); disk_cache::frame_key(content, key.frame, key.scale, key.view ^ opts_hash) } @@ -441,6 +450,10 @@ pub struct Frames { /// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout. pub last_ms: Arc>, content_keys: ContentKeys, + /// Bumped when a virtual file is replaced. Lookups key on it instead of statting footage. + virtual_epoch: Arc, + /// Size and content hash for paths with no native metadata. Called only on a cache miss. + virtual_id: Option Option<(u64, u64)> + Send + Sync>>, #[cfg(not(target_arch = "wasm32"))] content_key_jobs: Arc>, gpu_retired: Arc, @@ -477,6 +490,8 @@ impl Default for Frames { ctx: None, last_ms: Arc::new(Mutex::new(0.0)), content_keys: Arc::default(), + virtual_epoch: Arc::new(AtomicU64::new(0)), + virtual_id: None, #[cfg(not(target_arch = "wasm32"))] content_key_jobs: Arc::default(), gpu_retired: Arc::default(), @@ -768,9 +783,10 @@ impl Frames { if src.disk.is_none() && remote.is_none() { return vec![]; } + let epoch = self.virtual_epoch.load(Ordering::Relaxed); #[cfg(not(target_arch = "wasm32"))] let content = { - let known = self.content_keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&(base.revision, base.comp)).copied(); + let known = self.content_keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&(base.revision, base.comp, epoch)).copied(); let Some(content) = known else { self.request_content_key(src.project.clone(), base.revision, base.comp); return vec![]; @@ -778,7 +794,7 @@ impl Frames { content }; #[cfg(target_arch = "wasm32")] - let content = content_key(&self.content_keys, &src.project, base.revision, base.comp); + let content = content_key(&self.content_keys, &src.project, base.revision, base.comp, epoch, self.virtual_id.as_deref()); let ram: HashSet = self.cached_frames(base).into_iter().collect(); let oh = opts_hash(opts); (0..frames.clamp(0, 100_000)) @@ -803,6 +819,7 @@ impl Frames { jobs.running = true; drop(jobs); let (jobs, keys, ctx) = (self.content_key_jobs.clone(), self.content_keys.clone(), self.ctx.clone()); + let (epoch, id) = (self.virtual_epoch.clone(), self.virtual_id.clone()); let work = move || loop { let next = { let mut jobs = jobs.lock().unwrap_or_else(std::sync::PoisonError::into_inner); @@ -813,7 +830,9 @@ impl Frames { next }; let Some((project, revision, comp)) = next else { return }; - if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| content_key(&keys, &project, revision, comp))).is_err() { + let epoch = epoch.load(Ordering::Relaxed); + let id = id.clone(); + if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| content_key(&keys, &project, revision, comp, epoch, id.as_deref()))).is_err() { log::error!("could not compute a preview disk-cache key; the cache indicator will retry"); } if let Some(ctx) = &ctx { @@ -964,10 +983,22 @@ impl Frames { ctx: self.ctx.clone(), last: self.last_ms.clone(), content_keys: self.content_keys.clone(), + virtual_epoch: self.virtual_epoch.clone(), + virtual_id: self.virtual_id.clone(), gpu_retired: self.gpu_retired.clone(), } } + /// Browser file table: `epoch` bumps on each replacement, and `id` is the size and content + /// hash used when a content key is rebuilt. Neither runs on a cache hit. + pub fn set_virtual_media(&mut self, epoch: Arc, id: Arc Option<(u64, u64)> + Send + Sync>) { + self.virtual_epoch = epoch; + self.virtual_id = Some(id); + if let Ok(mut keys) = self.content_keys.lock() { + keys.clear(); + } + } + /// Render frames with `remote` from now on (`None`: on this thread again). pub fn set_remote(&mut self, remote: Option>) { self.remote = remote; @@ -1036,7 +1067,8 @@ impl Frames { let fw = self.worker(); let busy = self.remote_busy.clone(); let t0 = web_time::Instant::now(); - let disk_key = remote.disk().then(|| frame_disk_key(&w.content_keys, &job.src.project, &key, opts_hash(&job.opts))); + let epoch = w.virtual_epoch.load(Ordering::Relaxed); + let disk_key = remote.disk().then(|| frame_disk_key(&w.content_keys, &job.src.project, &key, opts_hash(&job.opts), epoch, w.virtual_id.as_deref())); let rj = RemoteJob { project: job.src.project.clone(), revision: key.revision, comp: job.comp, t: job.t, opts: job.opts, disk_key }; remote.start( rj, @@ -1093,6 +1125,8 @@ struct Worker { ctx: Option, last: Arc>, content_keys: ContentKeys, + virtual_epoch: Arc, + virtual_id: Option Option<(u64, u64)> + Send + Sync>>, gpu_retired: Arc, } @@ -1147,7 +1181,12 @@ impl Worker { job }; let t0 = web_time::Instant::now(); - let disk = job.src.disk.as_ref().map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts)))); + let epoch = self.virtual_epoch.load(Ordering::Relaxed); + let disk = job + .src + .disk + .as_ref() + .map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts), epoch, self.virtual_id.as_deref()))); let from_disk = disk.as_ref().and_then(|(dc, k)| dc.get_frame(*k)); let disk_hit = from_disk.is_some(); let ci = match from_disk { @@ -1428,7 +1467,7 @@ mod tests { assert_eq!(frames.content_key_jobs.lock().unwrap().pending.len(), 1); release.send(()).unwrap(); let until = Instant::now() + Duration::from_secs(3); - while !frames.content_keys.lock().unwrap().contains_key(&(base.revision, base.comp)) { + while !frames.content_keys.lock().unwrap().contains_key(&(base.revision, base.comp, 0)) { assert!(Instant::now() < until); std::thread::sleep(Duration::from_millis(1)); }