Key browser preview caches on a file generation, and hash bytes only when that key misses.

Replacing a virtual file bumps a counter the UI lookup reads. The content hash runs on that miss, from a callback, so the frame path does not stat every footage item or hash on every store write.

apps/effectcraft-web/src/app.rs

@@ -257,6 +257,7 @@ pub async fn start(canvas_id: String) -> Result<(), JsValue> {
                 if n > 0 {
                     app.frames.set_remote(Some(Arc::new(frames::WebFrames::start(n, !query_flag("nogpuworkers")))));
                 }
+                app.frames.set_virtual_media(files::STORE.media_epoch(), Arc::new(|path| files::STORE.file_identity(path)));
                 let (tx, rx) = std::sync::mpsc::channel();
                 api::set_sender(tx);
                 app = app.with_control(rx);

apps/effectcraft-web/src/frames.rs

@@ -416,6 +416,7 @@ fn post_reply(r: &FrameReply, px: Option<&[u8]>) {
 fn new_server() -> FrameServer {
     let mut s = FrameServer::new(crate::worker::pool(), Some(Arc::new(effectcraft_expr::Expressions)));
     s.set_layer_store(Some(Arc::new(effectcraft_engine::render::PrefetchStore::default())));
+    s.set_virtual_id(Some(Arc::new(|path| crate::files::STORE.file_identity(path))));
     s
 }
 

apps/effectcraft-web/src/store.rs

@@ -20,6 +20,7 @@
 
 use std::collections::BTreeMap;
 use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU64, Ordering};
 use std::sync::{Arc, Mutex, OnceLock};
 
 use effectcraft_engine::config::{ConfigStore, FileOps};
@@ -45,6 +46,8 @@ pub struct Entry {
     /// Which write this is: new with every write of the key, so a file replaced by another of
     /// the same size still reads as changed (workers are sent it again, #218).
     pub version: u64,
+    /// FNV-1a of `data`, filled the first time a disk-cache key asks for it.
+    hash: Option<u64>,
 }
 
 /// A queued write for the browser storage: `data` = `None` deletes the key.
@@ -69,13 +72,13 @@ impl Mirror {
     /// An entry read from the browser's storage at startup (nothing to flush).
     pub fn load(&mut self, key: &str, data: Arc<[u8]>, modified: f64) {
         let version = self.next_version();
-        self.entries.insert(key.to_string(), Entry { data, modified, persist: true, version });
+        self.entries.insert(key.to_string(), Entry { data, modified, persist: true, version, hash: None });
     }
 
     pub fn put(&mut self, key: &str, data: Arc<[u8]>, modified: f64, persist: bool) {
         let was_persisted = self.entries.get(key).is_some_and(|e| e.persist);
         let version = self.next_version();
-        self.entries.insert(key.to_string(), Entry { data, modified, persist, version });
+        self.entries.insert(key.to_string(), Entry { data, modified, persist, version, hash: None });
         if persist || was_persisted {
             self.touch(key);
         }
@@ -149,14 +152,31 @@ pub fn key_path(key: &str) -> Option<&str> {
     key.strip_prefix(FILES).filter(|p| p.starts_with('/'))
 }
 
+/// FNV-1a 64. Computed when a disk-cache key first needs the file, not on every write.
+fn content_hash(data: &[u8]) -> u64 {
+    let mut h: u64 = 0xcbf2_9ce4_8422_2325;
+    for &b in data {
+        h ^= u64::from(b);
+        h = h.wrapping_mul(0x0000_0100_0000_01b3);
+    }
+    h
+}
+
 /// The persistent store: a [`Mirror`] and a hook that wakes the flusher after changes.
-#[derive(Default)]
 pub struct Store {
     pub mirror: Mutex<Mirror>,
     /// Called after every change that needs flushing (the web host schedules a flush).
     pub on_change: OnceLock<fn()>,
     /// Clock in milliseconds since the Unix epoch (tests pin it).
     pub clock: OnceLock<fn() -> f64>,
+    /// Bumped when a virtual file is written. Preview cache lookups key on it; they do not hash.
+    media_epoch: Arc<AtomicU64>,
+}
+
+impl Default for Store {
+    fn default() -> Self {
+        Self { mirror: Mutex::default(), on_change: OnceLock::new(), clock: OnceLock::new(), media_epoch: Arc::new(AtomicU64::new(0)) }
+    }
 }
 
 impl Store {
@@ -179,12 +199,40 @@ impl Store {
     /// Store a file under `path` (persisted unless `persist` is false: render outputs).
     pub fn put_file(&self, path: &str, data: Arc<[u8]>, persist: bool) {
         let now = self.now();
-        self.lock().put(&file_key(path), data, now, persist);
+        let key = file_key(path);
+        let replaced = self.lock().get(&key).is_none_or(|e| !Arc::ptr_eq(&e.data, &data));
+        self.lock().put(&key, data, now, persist);
+        if replaced {
+            self.media_epoch.fetch_add(1, Ordering::Relaxed);
+        }
         if persist {
             self.changed();
         }
     }
 
+    /// The generation preview cache lookups key on. It changes when [`Self::put_file`] writes
+    /// a different blob, and it is not a content hash.
+    pub fn media_epoch(&self) -> Arc<AtomicU64> {
+        self.media_epoch.clone()
+    }
+
+    /// Size and content hash of the file under `path`. The hash is stored on the entry so a
+    /// later key does not walk the bytes again.
+    pub fn file_identity(&self, path: &str) -> Option<(u64, u64)> {
+        let key = file_key(path);
+        let mut mirror = self.lock();
+        let entry = mirror.entries.get(&key)?;
+        if let Some(hash) = entry.hash {
+            return Some((entry.data.len() as u64, hash));
+        }
+        let hash = content_hash(&entry.data);
+        let len = entry.data.len() as u64;
+        if let Some(entry) = mirror.entries.get_mut(&key) {
+            entry.hash = Some(hash);
+        }
+        Some((len, hash))
+    }
+
     pub fn file(&self, path: &str) -> Option<Arc<[u8]>> {
         self.lock().get(&file_key(path)).map(|e| e.data.clone())
     }
@@ -446,6 +494,11 @@ mod tests {
         assert_ne!(a, b);
         assert_eq!(&data[..], b"blue");
         assert_eq!(s.file_versioned("/collision.png").map(|f| f.1), Some(b), "reading doesn't change it");
+        let epoch = s.media_epoch().load(Ordering::Relaxed);
+        let id = s.file_identity("/collision.png").expect("identity");
+        assert_eq!(id, (4, content_hash(b"blue")));
+        assert_eq!(s.file_identity("/collision.png"), Some(id), "the hash is kept on the entry");
+        assert!(epoch >= 2, "each new blob bumps the generation the preview keys on");
     }
 
     /// A write that failed is flushed again with the key's latest state (#215).

crates/engine/src/remote.rs

@@ -263,6 +263,8 @@ pub struct FrameServer {
     /// The disk cache's layer buffers (see the module docs) and the project's salt.
     layer_store: Option<Arc<PrefetchStore>>,
     salt: u64,
+    /// Size and content hash for footage paths with no native metadata (the browser file table).
+    virtual_id: Option<Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>>,
 }
 
 /// A frame whose readbacks are in flight.
@@ -294,9 +296,16 @@ impl FrameServer {
             pending: None,
             layer_store: None,
             salt: 0,
+            virtual_id: None,
         }
     }
 
+    /// Identity of virtual footage files, used when native metadata cannot describe the path.
+    /// Called from [`FrameServer::load`], not from the UI thread's cache lookup.
+    pub fn set_virtual_id(&mut self, id: Option<Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>>) {
+        self.virtual_id = id;
+    }
+
     /// The store behind the layer cache when a request asks for `layers` (the browser: the
     /// disk cache's layer buffers, fetched before the request).
     pub fn set_layer_store(&mut self, store: Option<Arc<PrefetchStore>>) {
@@ -338,7 +347,7 @@ impl FrameServer {
 
     fn load(&mut self, revision: u64) -> Result<(), String> {
         let p = Project::from_json(&self.doc.to_string()).map_err(|e| e.to_string())?;
-        self.salt = effectcraft_render::disk_cache::footage_salt(&p);
+        self.salt = effectcraft_render::disk_cache::footage_salt_with(&p, self.virtual_id.as_deref());
         self.cache.set_store_salt(self.salt);
         self.project = Some(Arc::new(p));
         self.revision = Some(revision);

crates/render/src/disk_cache.rs

@@ -779,8 +779,12 @@ fn hash_debug(h: &mut Hash128, v: &impl std::fmt::Debug) {
     h.write(&[0xff]);
 }
 
-/// A file's size and modification time (so edited footage gets new keys).
-fn file_stamp(h: &mut Hash128, path: &str) {
+/// Size and content hash of a path that has no native metadata (the browser file table).
+pub type VirtualFileId<'a> = &'a (dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync);
+
+/// A file's size and modification time (so edited footage gets new keys). When the path has no
+/// native metadata, `virtual_id` supplies a durable size and content hash instead.
+fn file_stamp(h: &mut Hash128, path: &str, virtual_id: Option<VirtualFileId<'_>>) {
     if path.is_empty() {
         return;
     }
@@ -791,26 +795,36 @@ fn file_stamp(h: &mut Hash128, path: &str) {
         {
             h.write_u64(d.as_nanos() as u64);
         }
+        return;
+    }
+    if let Some((len, hash)) = virtual_id.and_then(|id| id(path)) {
+        h.write_u64(len);
+        h.write_u64(hash);
     }
 }
 
-fn footage_stamp(h: &mut Hash128, f: &effectcraft_project::Footage) {
-    file_stamp(h, &f.path);
+fn footage_stamp(h: &mut Hash128, f: &effectcraft_project::Footage, virtual_id: Option<VirtualFileId<'_>>) {
+    file_stamp(h, &f.path, virtual_id);
     for p in &f.sequence {
-        file_stamp(h, p);
+        file_stamp(h, p, virtual_id);
     }
 }
 
 /// Salt for layer cache keys on disk: the renderer version and every footage file's identity
 /// (layer keys name footage items, not file contents).
 pub fn footage_salt(project: &effectcraft_project::Project) -> u64 {
+    footage_salt_with(project, None)
+}
+
+/// [`footage_salt`] with [`VirtualFileId`] for paths `std::fs::metadata` cannot describe.
+pub fn footage_salt_with(project: &effectcraft_project::Project, virtual_id: Option<VirtualFileId<'_>>) -> u64 {
     let mut h = Hash128::default();
     h.write(RENDER_VERSION.as_bytes());
     for (id, it) in &project.items {
         if let effectcraft_project::ItemKind::Footage(f) = &it.kind {
             h.write_u64(id.0);
             hash_debug(&mut h, f);
-            footage_stamp(&mut h, f);
+            footage_stamp(&mut h, f, virtual_id);
         }
     }
     let v = h.finish();
@@ -822,6 +836,11 @@ pub fn footage_salt(project: &effectcraft_project::Project) -> u64 {
 /// files' size and time). With expressions in play (which can read any comp) the whole project
 /// counts. Switches that don't change pixels (Audio, Lock, Shy, Hide Shy Layers) don't count.
 pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId) -> u128 {
+    comp_content_key_with(project, comp, None)
+}
+
+/// [`comp_content_key`] with [`VirtualFileId`] for paths `std::fs::metadata` cannot describe.
+pub fn comp_content_key_with(project: &effectcraft_project::Project, comp: effectcraft_project::ItemId, virtual_id: Option<VirtualFileId<'_>>) -> u128 {
     use effectcraft_project::{ItemKind, LayerSource};
     let mut h = Hash128::default();
     h.write(RENDER_VERSION.as_bytes());
@@ -855,11 +874,11 @@ pub fn comp_content_key(project: &effectcraft_project::Project, comp: effectcraf
             k => hash_debug(&mut h, k),
         }
         if let ItemKind::Footage(f) = &it.kind {
-            footage_stamp(&mut h, f);
+            footage_stamp(&mut h, f, virtual_id);
         }
         hash_debug(&mut h, &it.proxy);
         if let Some(px) = &it.proxy {
-            footage_stamp(&mut h, &px.footage);
+            footage_stamp(&mut h, &px.footage, virtual_id);
         }
     }
     h.write_u64(comp.0);
@@ -1082,4 +1101,29 @@ mod tests {
             assert_eq!(b.get_layer(k).unwrap().img, buf(k as u32, 16, 16).img);
         }
     }
+
+    /// A path with no native metadata takes its key from the caller's size and content hash (#1253).
+    #[test]
+    fn virtual_file_bytes_change_content_keys() {
+        use effectcraft_project::build;
+        use effectcraft_project::{Comp, Footage, ItemKind, LayerSource, Project};
+        use effectcraft_time::{FrameRate, Tick};
+
+        let mut p = Project::default();
+        let f = Footage { path: "/virtual.mov".into(), width: 16, height: 9, duration: Tick::from_seconds_f64(1.0), has_video: true, ..Default::default() };
+        let fid = p.add_item("virtual.mov", effectcraft_color::Label::None, None, ItemKind::Footage(f));
+        let mut comp = Comp::new(16, 9, FrameRate::FPS_30, Tick::from_seconds_f64(1.0));
+        comp.layers = vec![build::layer(&mut p, &comp, "clip", LayerSource::Footage { item: fid }, (16, 9), None)];
+        let cid = p.add_item("C", effectcraft_color::Label::None, None, ItemKind::Comp(comp.into()));
+
+        let id = |mark: u64| move |path: &str| (path == "/virtual.mov").then_some((8, mark));
+        let (one, two) = (id(1), id(2));
+        let a = footage_salt_with(&p, Some(&one));
+        let b = footage_salt_with(&p, Some(&two));
+        let ka = comp_content_key_with(&p, cid, Some(&one));
+        let kb = comp_content_key_with(&p, cid, Some(&two));
+        assert_ne!(a, b, "replaced virtual media must not reuse the salt");
+        assert_ne!(ka, kb, "replaced virtual media must not reuse the frame key");
+        assert_eq!(comp_content_key(&p, cid), comp_content_key_with(&p, cid, None));
+    }
 }

crates/ui-egui/src/frames.rs

@@ -24,7 +24,7 @@
 //! them (Info panel, eyedroppers, histograms).
 
 use std::collections::{BTreeSet, HashMap, HashSet, VecDeque};
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 use std::sync::{Arc, Mutex};
 
 use effectcraft_engine::project::{Comp, ItemId, ItemKind, LayerSource, Node, Project, PropGroup};
@@ -258,8 +258,10 @@ pub struct RenderSource {
 /// ([`Frames::identity`]).
 type Identity = (u64, Option<Arc<Project>>);
 
-/// Content keys of (project revision, comp), computed once per revision.
-type ContentKeys = Arc<Mutex<HashMap<(u64, u64), u128>>>;
+/// Content keys of (project revision, comp, virtual-file generation). The generation is one
+/// counter bumped when a browser file is replaced, so a lookup stays a map read. Hashing the
+/// new bytes happens only when that generation misses.
+type ContentKeys = Arc<Mutex<HashMap<(u64, u64, u64), u128>>>;
 
 #[cfg(not(target_arch = "wasm32"))]
 #[derive(Default)]
@@ -268,16 +270,16 @@ struct ContentKeyJobs {
     pending: VecDeque<(Arc<Project>, u64, u64)>,
 }
 
-fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64) -> u128 {
-    if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp)).copied()) {
+fn content_key(keys: &ContentKeys, project: &Project, revision: u64, comp: u64, epoch: u64, virtual_id: Option<disk_cache::VirtualFileId<'_>>) -> u128 {
+    if let Some(k) = keys.lock().ok().and_then(|m| m.get(&(revision, comp, epoch)).copied()) {
         return k;
     }
-    let k = disk_cache::comp_content_key(project, ItemId(comp));
+    let k = disk_cache::comp_content_key_with(project, ItemId(comp), virtual_id);
     if let Ok(mut m) = keys.lock() {
         if m.len() > 256 {
             m.clear();
         }
-        m.insert((revision, comp), k);
+        m.insert((revision, comp, epoch), k);
     }
     k
 }
@@ -363,8 +365,15 @@ pub fn opts_hash(o: &RenderOpts) -> u64 {
 }
 
 /// Disk key of a viewer frame (render options included: region of interest, draft…).
-fn frame_disk_key(keys: &ContentKeys, project: &Project, key: &FrameKey, opts_hash: u64) -> u128 {
-    let content = content_key(keys, project, key.revision, key.comp);
+fn frame_disk_key(
+    keys: &ContentKeys,
+    project: &Project,
+    key: &FrameKey,
+    opts_hash: u64,
+    epoch: u64,
+    virtual_id: Option<disk_cache::VirtualFileId<'_>>,
+) -> u128 {
+    let content = content_key(keys, project, key.revision, key.comp, epoch, virtual_id);
     disk_cache::frame_key(content, key.frame, key.scale, key.view ^ opts_hash)
 }
 
@@ -441,6 +450,10 @@ pub struct Frames {
     /// Render time of the last viewer (urgent) frame in ms, for the Info panel / perf readout.
     pub last_ms: Arc<Mutex<f64>>,
     content_keys: ContentKeys,
+    /// Bumped when a virtual file is replaced. Lookups key on it instead of statting footage.
+    virtual_epoch: Arc<AtomicU64>,
+    /// Size and content hash for paths with no native metadata. Called only on a cache miss.
+    virtual_id: Option<Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>>,
     #[cfg(not(target_arch = "wasm32"))]
     content_key_jobs: Arc<Mutex<ContentKeyJobs>>,
     gpu_retired: Arc<AtomicBool>,
@@ -477,6 +490,8 @@ impl Default for Frames {
             ctx: None,
             last_ms: Arc::new(Mutex::new(0.0)),
             content_keys: Arc::default(),
+            virtual_epoch: Arc::new(AtomicU64::new(0)),
+            virtual_id: None,
             #[cfg(not(target_arch = "wasm32"))]
             content_key_jobs: Arc::default(),
             gpu_retired: Arc::default(),
@@ -768,9 +783,10 @@ impl Frames {
         if src.disk.is_none() && remote.is_none() {
             return vec![];
         }
+        let epoch = self.virtual_epoch.load(Ordering::Relaxed);
         #[cfg(not(target_arch = "wasm32"))]
         let content = {
-            let known = self.content_keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&(base.revision, base.comp)).copied();
+            let known = self.content_keys.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&(base.revision, base.comp, epoch)).copied();
             let Some(content) = known else {
                 self.request_content_key(src.project.clone(), base.revision, base.comp);
                 return vec![];
@@ -778,7 +794,7 @@ impl Frames {
             content
         };
         #[cfg(target_arch = "wasm32")]
-        let content = content_key(&self.content_keys, &src.project, base.revision, base.comp);
+        let content = content_key(&self.content_keys, &src.project, base.revision, base.comp, epoch, self.virtual_id.as_deref());
         let ram: HashSet<i64> = self.cached_frames(base).into_iter().collect();
         let oh = opts_hash(opts);
         (0..frames.clamp(0, 100_000))
@@ -803,6 +819,7 @@ impl Frames {
         jobs.running = true;
         drop(jobs);
         let (jobs, keys, ctx) = (self.content_key_jobs.clone(), self.content_keys.clone(), self.ctx.clone());
+        let (epoch, id) = (self.virtual_epoch.clone(), self.virtual_id.clone());
         let work = move || loop {
             let next = {
                 let mut jobs = jobs.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
@@ -813,7 +830,9 @@ impl Frames {
                 next
             };
             let Some((project, revision, comp)) = next else { return };
-            if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| content_key(&keys, &project, revision, comp))).is_err() {
+            let epoch = epoch.load(Ordering::Relaxed);
+            let id = id.clone();
+            if std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| content_key(&keys, &project, revision, comp, epoch, id.as_deref()))).is_err() {
                 log::error!("could not compute a preview disk-cache key; the cache indicator will retry");
             }
             if let Some(ctx) = &ctx {
@@ -964,10 +983,22 @@ impl Frames {
             ctx: self.ctx.clone(),
             last: self.last_ms.clone(),
             content_keys: self.content_keys.clone(),
+            virtual_epoch: self.virtual_epoch.clone(),
+            virtual_id: self.virtual_id.clone(),
             gpu_retired: self.gpu_retired.clone(),
         }
     }
 
+    /// Browser file table: `epoch` bumps on each replacement, and `id` is the size and content
+    /// hash used when a content key is rebuilt. Neither runs on a cache hit.
+    pub fn set_virtual_media(&mut self, epoch: Arc<AtomicU64>, id: Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>) {
+        self.virtual_epoch = epoch;
+        self.virtual_id = Some(id);
+        if let Ok(mut keys) = self.content_keys.lock() {
+            keys.clear();
+        }
+    }
+
     /// Render frames with `remote` from now on (`None`: on this thread again).
     pub fn set_remote(&mut self, remote: Option<Arc<dyn RemoteFrames>>) {
         self.remote = remote;
@@ -1036,7 +1067,8 @@ impl Frames {
             let fw = self.worker();
             let busy = self.remote_busy.clone();
             let t0 = web_time::Instant::now();
-            let disk_key = remote.disk().then(|| frame_disk_key(&w.content_keys, &job.src.project, &key, opts_hash(&job.opts)));
+            let epoch = w.virtual_epoch.load(Ordering::Relaxed);
+            let disk_key = remote.disk().then(|| frame_disk_key(&w.content_keys, &job.src.project, &key, opts_hash(&job.opts), epoch, w.virtual_id.as_deref()));
             let rj = RemoteJob { project: job.src.project.clone(), revision: key.revision, comp: job.comp, t: job.t, opts: job.opts, disk_key };
             remote.start(
                 rj,
@@ -1093,6 +1125,8 @@ struct Worker {
     ctx: Option<egui::Context>,
     last: Arc<Mutex<f64>>,
     content_keys: ContentKeys,
+    virtual_epoch: Arc<AtomicU64>,
+    virtual_id: Option<Arc<dyn Fn(&str) -> Option<(u64, u64)> + Send + Sync>>,
     gpu_retired: Arc<AtomicBool>,
 }
 
@@ -1147,7 +1181,12 @@ impl Worker {
             job
         };
         let t0 = web_time::Instant::now();
-        let disk = job.src.disk.as_ref().map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts))));
+        let epoch = self.virtual_epoch.load(Ordering::Relaxed);
+        let disk = job
+            .src
+            .disk
+            .as_ref()
+            .map(|dc| (dc.clone(), frame_disk_key(content_keys, &job.src.project, &job.key, opts_hash(&job.opts), epoch, self.virtual_id.as_deref())));
         let from_disk = disk.as_ref().and_then(|(dc, k)| dc.get_frame(*k));
         let disk_hit = from_disk.is_some();
         let ci = match from_disk {
@@ -1428,7 +1467,7 @@ mod tests {
         assert_eq!(frames.content_key_jobs.lock().unwrap().pending.len(), 1);
         release.send(()).unwrap();
         let until = Instant::now() + Duration::from_secs(3);
-        while !frames.content_keys.lock().unwrap().contains_key(&(base.revision, base.comp)) {
+        while !frames.content_keys.lock().unwrap().contains_key(&(base.revision, base.comp, 0)) {
             assert!(Instant::now() < until);
             std::thread::sleep(Duration::from_millis(1));
         }