M13.30: web depth and polish

- GPU in browser job workers: each job worker opens its own WebGPU device
  (deferred readbacks); the Render Queue export and the analyses' frame loops
  are futures (render::passes, offload::run_request_async) whose frames
  render in passes, awaiting the device between them. GPU particles and
  Advanced 3D read back under keys too. workers.jobs reports passes and
  readbacks; the smoke test measures them.
- Layer buffers in the browser's OPFS disk cache: frame workers back their
  layer caches with a PrefetchStore; the page plans what to fetch before a
  request from recent misses (LayerPrefetch over the shared DiskIndex).
- Templates embed footage (default on, size-capped with a warning) and
  creating from a template extracts it next to the new project / into
  browser storage.
- ICC LUT profiles (lut8/lut16/lutAtoB/lutBtoA, from the public ICC spec) in
  Simulate Output > My Custom RGB, baked into a 3D LUT for the viewer.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

apps/effectcraft-web/src/app.rs

@@ -141,7 +141,11 @@ pub fn session() -> Session {
         script: Some(effectcraft_host::script::runner),
         exporter: Some(Arc::new(effectcraft_host::FileExporter { sink: Some(files::export_sink()) })),
         config: Some(Arc::new(persist::config())),
-        offload: (!query_flag("noworkers")).then(|| Arc::new(worker::WorkerOffload) as Arc<dyn effectcraft_engine::offload::Offload>),
+        offload: (!query_flag("noworkers")).then(|| {
+            // Job workers render on a WebGPU device of their own (`?nogpuworkers`: their CPU).
+            worker::set_job_worker_gpu(!query_flag("nogpuworkers"));
+            Arc::new(worker::WorkerOffload) as Arc<dyn effectcraft_engine::offload::Offload>
+        }),
         browser: Some(Arc::new(browse::WebBrowser)),
         storage: Some(Arc::new(storage::WebStorage)),
         ..Default::default()

apps/effectcraft-web/src/diskcache.rs

@@ -13,13 +13,18 @@
 //! - **Reads** are asynchronous: a frame whose key is indexed is read and decoded here instead
 //!   of rendering it (`crate::frames`); a damaged or missing file is a miss and is forgotten.
 //!
-//! Only frames are cached on disk in the browser (the desktop also keeps slow layer buffers):
-//! a layer lookup happens in the middle of a synchronous render, where the browser's file
-//! system can't be read.
+//! - **Layer buffers** too (`layers/<32 hex>.ecc`): a layer lookup happens in the middle of a
+//!   synchronous render, where the browser's file system can't be read, so the frame workers
+//!   back their layer caches with a `PrefetchStore`. They report each frame's layer-cache
+//!   misses; before the next request the page names the recent misses that are on disk
+//!   ([`layer_prefetch`], `disk_cache::LayerPrefetch`), and the worker reads those entries
+//!   into memory before rendering, so the lookups find them. Slow buffers are written by the
+//!   worker after its frame and indexed here like frames (one least-recently-used order for
+//!   both kinds).
 
 use std::cell::RefCell;
 
-use effectcraft_engine::render::disk_cache::{self, DiskIndex, Kind};
+use effectcraft_engine::render::disk_cache::{self, DiskIndex, Kind, LayerPrefetch};
 use serde_json::{Value, json};
 use wasm_bindgen::prelude::*;
 use wasm_bindgen_futures::JsFuture;
@@ -29,7 +34,7 @@ extern "C" {
     #[wasm_bindgen(js_name = cacheList)]
     fn cache_list() -> js_sys::Promise;
     #[wasm_bindgen(js_name = cacheRead)]
-    fn cache_read(name: &str) -> js_sys::Promise;
+    pub(crate) fn cache_read(name: &str) -> js_sys::Promise;
     #[wasm_bindgen(js_name = cacheWrite)]
     pub(crate) fn cache_write(name: &str, bytes: js_sys::Uint8Array) -> js_sys::Promise;
     #[wasm_bindgen(js_name = cacheDelete)]
@@ -53,6 +58,10 @@ struct State {
     writes: u64,
     evictions: u64,
     error: Option<String>,
+    /// Recent layer-cache misses of the frame workers (what to prefetch).
+    prefetch: LayerPrefetch,
+    layer_hits: u64,
+    layer_writes: u64,
 }
 
 thread_local! {
@@ -68,19 +77,19 @@ pub fn init() {
     wasm_bindgen_futures::spawn_local(async {
         match JsFuture::from(cache_list()).await {
             Ok(list) => {
-                let mut entries: Vec<(f64, u128, u64)> = js_sys::Array::from(&list)
+                let mut entries: Vec<(f64, Kind, u128, u64)> = js_sys::Array::from(&list)
                     .iter()
                     .filter_map(|e| {
                         let get = |k: &str| js_sys::Reflect::get(&e, &k.into()).ok();
-                        let key = disk_cache::parse_entry_name(&get("name")?.as_string()?)?;
-                        Some((get("modified")?.as_f64().unwrap_or(0.0), key, get("size")?.as_f64().unwrap_or(0.0) as u64))
+                        let (kind, key) = parse_name(&get("name")?.as_string()?)?;
+                        Some((get("modified")?.as_f64().unwrap_or(0.0), kind, key, get("size")?.as_f64().unwrap_or(0.0) as u64))
                     })
                     .collect();
                 entries.sort_by(|a, b| a.0.total_cmp(&b.0));
                 ST.with(|s| {
                     let mut s = s.borrow_mut();
-                    for (_, key, bytes) in entries {
-                        s.index.insert(Kind::Frame, key, bytes);
+                    for (_, kind, key, bytes) in entries {
+                        s.index.insert(kind, key, bytes);
                     }
                     s.loaded = true;
                     s.available = true;
@@ -96,6 +105,46 @@ pub fn init() {
     });
 }
 
+/// An entry's name in `js/host.js`'s cache folders (`layers/` for layer buffers).
+pub(crate) fn name_of(kind: Kind, key: u128) -> String {
+    match kind {
+        Kind::Frame => disk_cache::entry_name(key),
+        Kind::Layer => format!("layers/{}", disk_cache::entry_name(key)),
+    }
+}
+
+fn parse_name(name: &str) -> Option<(Kind, u128)> {
+    match name.strip_prefix("layers/") {
+        Some(n) => Some((Kind::Layer, disk_cache::parse_entry_name(n)?)),
+        None => Some((Kind::Frame, disk_cache::parse_entry_name(name)?)),
+    }
+}
+
+/// Layer entries (32 hex digits) a frame worker should read before its next frame: recent
+/// misses that are on disk.
+pub fn layer_prefetch(max: usize) -> Vec<String> {
+    if !active() {
+        return vec![];
+    }
+    ST.with(|s| {
+        let s = s.borrow();
+        s.prefetch.plan(&s.index, max).into_iter().map(|k| format!("{k:032x}")).collect()
+    })
+}
+
+/// A frame worker's layer-cache misses and the buffers it served from what it fetched.
+pub fn note_layers(misses: &[String], hits: &[String]) {
+    let parse = |v: &[String]| v.iter().filter_map(|k| u128::from_str_radix(k, 16).ok()).collect::<Vec<u128>>();
+    ST.with(|s| {
+        let mut s = s.borrow_mut();
+        s.prefetch.note_misses(parse(misses));
+        for k in parse(hits) {
+            s.index.touch(Kind::Layer, k);
+            s.layer_hits += 1;
+        }
+    });
+}
+
 /// Apply the settings: on/off and the limit (`max_gb`, at most half the origin's quota when it
 /// is known).
 pub fn configure(enabled: bool, max_gb: u32, quota: Option<u64>) {
@@ -155,12 +204,15 @@ pub async fn read(key: u128) -> Option<disk_cache::Frame8> {
     frame
 }
 
-/// A worker stored the frame of `key` (`bytes` on disk).
-pub fn stored(key: u128, bytes: u64) {
+/// A worker stored the frame (or, with `layer`, the layer buffer) of `key` (`bytes` on disk).
+pub fn stored(key: u128, bytes: u64, layer: bool) {
     ST.with(|s| {
         let mut s = s.borrow_mut();
-        s.index.insert(Kind::Frame, key, bytes);
+        s.index.insert(if layer { Kind::Layer } else { Kind::Frame }, key, bytes);
         s.writes += 1;
+        if layer {
+            s.layer_writes += 1;
+        }
     });
     evict();
 }
@@ -174,7 +226,7 @@ fn evict() {
         v
     });
     if !victims.is_empty() {
-        delete(victims.into_iter().map(|(_, k)| disk_cache::entry_name(k)).collect());
+        delete(victims.into_iter().map(|(kind, k)| name_of(kind, k)).collect());
     }
 }
 
@@ -194,6 +246,7 @@ pub fn clear() -> (u64, u64) {
         let mut s = s.borrow_mut();
         let r = (s.index.len() as u64, s.index.total());
         s.index.clear();
+        s.prefetch.clear();
         r
     });
     wasm_bindgen_futures::spawn_local(async {
@@ -212,10 +265,12 @@ pub fn stats() -> Value {
             "enabled": s.enabled && s.available,
             "available": s.available,
             "loaded": s.loaded,
-            "folder": "Origin Private File System: effectcraft-cache/v1/frames",
+            "folder": "Origin Private File System: effectcraft-cache/v1 (frames, layers)",
             "entries": s.index.len(),
-            "frames": s.index.len(),
-            "layers": 0,
+            "frames": s.index.keys(Kind::Frame).len(),
+            "layers": s.index.keys(Kind::Layer).len(),
+            "layerHits": s.layer_hits,
+            "layerWrites": s.layer_writes,
             "bytes": s.index.total(),
             "maxBytes": s.max_bytes,
             "hits": s.hits,

apps/effectcraft-web/src/files.rs

@@ -124,6 +124,10 @@ impl Services for WebServices {
         }
         download(file_name(path), data).map_err(|e| std::io::Error::other(format!("download failed: {e:?}")))
     }
+    fn store_file(&self, path: &str, data: &[u8]) -> std::io::Result<()> {
+        put(path, data.into());
+        Ok(())
+    }
 }
 
 /// Media import from the table (the bytes are also handed to the media pool, which decodes them).
@@ -137,6 +141,9 @@ impl effectcraft_engine::Importer for WebImporter {
         self.pool.add_bytes(path, bytes.clone());
         effectcraft_media::probe_bytes(path, bytes).map_err(|e| e.to_string())
     }
+    fn register(&self, path: &str, data: &[u8]) {
+        self.pool.add_bytes(path, get(path).unwrap_or_else(|| data.into()));
+    }
 }
 
 /// Render Queue output: files land in the table and are downloaded after the render

apps/effectcraft-web/src/frames.rs

@@ -38,7 +38,7 @@ extern "C" {
     #[wasm_bindgen(js_name = frameWorkerPost)]
     fn frame_worker_post(index: u32, msg: &JsValue, transfer: &js_sys::Array);
     #[wasm_bindgen(js_name = hasWebGpu)]
-    fn has_web_gpu() -> bool;
+    pub(crate) fn has_web_gpu() -> bool;
 }
 
 /// One frame worker as the page sees it.
@@ -134,7 +134,10 @@ impl WebFrames {
                 frame_worker_post(index, &obj(&[("type", "frame".into()), ("json", t.into())]), &js_sys::Array::new());
             }
             let disk = job.disk_key.map(|k| format!("{k:032x}"));
-            let req = FrameMsg::Render { id, revision: job.revision, comp: job.comp, time: job.t, opts: Box::new(job.opts), disk };
+            // Layer buffers from the disk cache: what to read before rendering.
+            let layers = crate::diskcache::active();
+            let prefetch = crate::diskcache::layer_prefetch(64);
+            let req = FrameMsg::Render { id, revision: job.revision, comp: job.comp, time: job.t, opts: Box::new(job.opts), disk, layers, prefetch };
             frame_worker_post(
                 index,
                 &obj(&[("type", "frame".into()), ("json", serde_json::to_string(&req).unwrap_or_default().into())]),
@@ -210,7 +213,8 @@ fn on_message(index: u32, json: JsValue, bytes: JsValue, error: JsValue) {
         }
         let reply: FrameReply = serde_json::from_str(&json.as_string()?).ok()?;
         match reply {
-            FrameReply::Frame { id, width, height, ms, passes, gpu } => {
+            FrameReply::Frame { id, width, height, ms, passes, gpu, layer_misses, layer_hits } => {
+                crate::diskcache::note_layers(&layer_misses, &layer_hits);
                 let (_, d) = w.busy.take_if(|b| b.0 == id)?;
                 let rgba = bytes.dyn_into::<js_sys::Uint8Array>().map(|b| b.to_vec()).unwrap_or_default();
                 st.rendered += 1;
@@ -231,9 +235,9 @@ fn on_message(index: u32, json: JsValue, bytes: JsValue, error: JsValue) {
                 w.mirror.reset();
                 None
             }
-            FrameReply::Stored { key, bytes } => {
+            FrameReply::Stored { key, bytes, layer } => {
                 if let Ok(k) = u128::from_str_radix(&key, 16) {
-                    crate::diskcache::stored(k, bytes);
+                    crate::diskcache::stored(k, bytes, layer);
                 }
                 None
             }
@@ -410,7 +414,9 @@ fn post_reply(r: &FrameReply, px: Option<&[u8]>) {
 }
 
 fn new_server() -> FrameServer {
-    FrameServer::new(crate::worker::pool(), Some(Arc::new(effectcraft_expr::Expressions)))
+    let mut s = FrameServer::new(crate::worker::pool(), Some(Arc::new(effectcraft_expr::Expressions)));
+    s.set_layer_store(Some(Arc::new(effectcraft_engine::render::PrefetchStore::default())));
+    s
 }
 
 /// Set up a frame worker (called once by `web/worker.js` after `workerInit`): its frame
@@ -450,9 +456,14 @@ pub async fn worker_frame(json: String) {
             return;
         }
     };
-    let disk = match &msg {
-        FrameMsg::Render { disk, .. } => disk.clone(),
-        _ => None,
+    let (disk, layers) = match &msg {
+        FrameMsg::Render { disk, layers, prefetch, .. } => {
+            if *layers {
+                prefetch_layers(prefetch).await;
+            }
+            (disk.clone(), *layers)
+        }
+        _ => (None, false),
     };
     let mut replies = SERVER.with(|c| c.borrow_mut().get_or_insert_with(new_server).handle(msg));
     while SERVER.with(|c| c.borrow().as_ref().is_some_and(|s| s.waiting())) {
@@ -466,9 +477,41 @@ pub async fn worker_frame(json: String) {
             let entry = effectcraft_engine::render::disk_cache::frame_entry(*width, *height, px);
             let name = format!("{key}.ecc");
             match JsFuture::from(crate::diskcache::cache_write(&name, js_sys::Uint8Array::from(&entry[..]))).await {
-                Ok(n) => post_reply(&FrameReply::Stored { key: key.clone(), bytes: n.as_f64().unwrap_or(entry.len() as f64) as u64 }, None),
+                Ok(n) => post_reply(&FrameReply::Stored { key: key.clone(), bytes: n.as_f64().unwrap_or(entry.len() as f64) as u64, layer: false }, None),
                 Err(e) => log::warn!("frame worker: disk cache write failed: {e:?}"),
             }
         }
     }
+    if layers {
+        store_layers().await;
+    }
+}
+
+/// Read the layer entries the page named into the server's store (those it lacks).
+async fn prefetch_layers(keys: &[String]) {
+    let Some(store) = SERVER.with(|c| c.borrow().as_ref().and_then(|s| s.layer_store().cloned())) else { return };
+    for k in keys {
+        let Ok(key) = u128::from_str_radix(k, 16) else { continue };
+        if store.has(key) {
+            continue;
+        }
+        let name = crate::diskcache::name_of(effectcraft_engine::render::disk_cache::Kind::Layer, key);
+        if let Ok(b) = JsFuture::from(crate::diskcache::cache_read(&name)).await
+            && let Ok(b) = b.dyn_into::<js_sys::Uint8Array>()
+        {
+            store.provide(key, &b.to_vec());
+        }
+    }
+}
+
+/// Write the slow layer buffers of the frame just rendered and report them.
+async fn store_layers() {
+    let Some(store) = SERVER.with(|c| c.borrow().as_ref().and_then(|s| s.layer_store().cloned())) else { return };
+    for (key, entry) in store.take_writes() {
+        let name = crate::diskcache::name_of(effectcraft_engine::render::disk_cache::Kind::Layer, key);
+        match JsFuture::from(crate::diskcache::cache_write(&name, js_sys::Uint8Array::from(&entry[..]))).await {
+            Ok(n) => post_reply(&FrameReply::Stored { key: format!("{key:032x}"), bytes: n.as_f64().unwrap_or(entry.len() as f64) as u64, layer: true }, None),
+            Err(e) => log::warn!("frame worker: layer cache write failed: {e:?}"),
+        }
+    }
 }

apps/effectcraft-web/src/worker.rs

@@ -9,10 +9,14 @@
 //! segmentations a propagation job sends back are relayed to them.
 //!
 //! **Worker side** — `web/worker.js` instantiates the same wasm module (the page's compiled
-//! `WebAssembly.Module`, so nothing is compiled twice) and calls [`worker_init`], then
+//! `WebAssembly.Module`, so nothing is compiled twice) and calls [`worker_init`] and
+//! [`worker_job_init`] (the worker's own WebGPU device, where workers have WebGPU), then
 //! [`worker_file`] per footage file and [`worker_job`] per job: a plain engine session (footage
-//! from memory, expressions, the exporter) runs it blocking and posts replies as it goes. Each
-//! worker has its own memory: no shared-memory threads, so this works on stable Rust.
+//! from memory, expressions, the exporter) runs it and posts replies as it goes. With a device,
+//! the job's frames (Render Queue frames, analysis input frames, particles, Advanced 3D) render
+//! on it in passes, the job awaiting the GPU's readbacks between them
+//! ([`effectcraft_engine::offload::run_request_async`]). Each worker has its own memory: no
+//! shared-memory threads, so this works on stable Rust.
 
 use std::cell::RefCell;
 use std::collections::HashMap;
@@ -32,6 +36,8 @@ extern "C" {
     fn worker_cancel(id: f64);
     #[wasm_bindgen(js_name = workerStats)]
     pub fn stats() -> JsValue;
+    #[wasm_bindgen(js_name = setJobWorkerGpu)]
+    pub fn set_job_worker_gpu(on: bool);
 }
 
 type OnMessage = Closure<dyn FnMut(String, JsValue, JsValue, JsValue)>;
@@ -42,6 +48,8 @@ thread_local! {
     /// The worker's session and its media pool.
     static SESSION: RefCell<Option<Session>> = const { RefCell::new(None) };
     static POOL: RefCell<Option<Arc<effectcraft_media::MediaPool>>> = const { RefCell::new(None) };
+    /// The job worker's GPU device (deferred readbacks), when it has one.
+    static GPU: RefCell<Option<effectcraft_gpu::Gpu>> = const { RefCell::new(None) };
 }
 
 /// The page's offload: one worker per running job (idle workers are reused).
@@ -169,9 +177,42 @@ pub fn worker_file(path: String, bytes: js_sys::Uint8Array) {
     });
 }
 
-/// Run one job (a JSON [`WorkerRequest`]) and post its replies.
+/// A job worker's GPU (called once by `web/worker.js` after `workerInit`): with `gpu` and
+/// WebGPU in workers, its own device with deferred readbacks becomes the session's
+/// accelerator. Posts `{type: "gpu", gpu?, error?}`.
+#[wasm_bindgen(js_name = workerJobInit)]
+pub async fn worker_job_init(gpu: bool) {
+    let r = if !gpu {
+        Err("GPU job workers are off (?nogpuworkers)".to_string())
+    } else if !crate::frames::has_web_gpu() {
+        Err("no WebGPU in workers".to_string())
+    } else {
+        effectcraft_gpu::Gpu::request_deferred().await
+    };
+    let msg = match r {
+        Ok(g) => {
+            let name = effectcraft_engine::render::Accelerator::name(&g);
+            SESSION.with(|c| {
+                if let Some(s) = c.borrow_mut().as_mut() {
+                    s.accel = Some(Arc::new(g.clone()));
+                }
+            });
+            GPU.with(|c| *c.borrow_mut() = Some(g));
+            log::info!("job worker: GPU {name}");
+            obj(&[("type", "gpu".into()), ("gpu", name.into())])
+        }
+        Err(e) => {
+            log::info!("job worker: CPU only ({e})");
+            obj(&[("type", "gpu".into()), ("error", e.into())])
+        }
+    };
+    post(&msg, None);
+}
+
+/// Run one job (a JSON [`WorkerRequest`]) and post its replies, then `{type: "stats", gpu,
+/// passes, readbacks, ms}` (the device's render passes and readbacks during the job).
 #[wasm_bindgen(js_name = workerJob)]
-pub fn worker_job(json: String) {
+pub async fn worker_job(json: String) {
     let send: effectcraft_engine::offload::Post = Rc::new(|r: WorkerReply| {
         let done = matches!(r, WorkerReply::Done);
         let t = serde_json::to_string(&r).unwrap_or_default();
@@ -185,13 +226,37 @@ pub fn worker_job(json: String) {
             return;
         }
     };
-    SESSION.with(|c| {
-        let mut c = c.borrow_mut();
-        let Some(s) = c.as_mut() else {
-            send(WorkerReply::Failed { error: "worker not initialised".into() });
-            send(WorkerReply::Done);
-            return;
-        };
-        effectcraft_engine::offload::run_request(s, req, &send);
+    // The session leaves its slot while the job runs (the job awaits the GPU in between;
+    // messages are handled one at a time, so nothing else needs it meanwhile).
+    let Some(mut s) = SESSION.with(|c| c.borrow_mut().take()) else {
+        send(WorkerReply::Failed { error: "worker not initialised".into() });
+        send(WorkerReply::Done);
+        return;
+    };
+    let gpu = GPU.with(|c| c.borrow().clone());
+    let before = gpu.as_ref().and_then(|g| g.deferred_stats()).unwrap_or_default();
+    let t0 = web_time::Instant::now();
+    let kind = format!("{:?}", req.job.kind());
+    // `Done` is posted last by the job; the stats go out just before it.
+    let stats_sent = Rc::new(std::cell::Cell::new(false));
+    let (g2, s2) = (gpu.clone(), stats_sent.clone());
+    let send2: effectcraft_engine::offload::Post = Rc::new(move |r: WorkerReply| {
+        if matches!(r, WorkerReply::Done) && !s2.replace(true) {
+            let after = g2.as_ref().and_then(|g| g.deferred_stats()).unwrap_or_default();
+            post(
+                &obj(&[
+                    ("type", "stats".into()),
+                    ("kind", kind.as_str().into()),
+                    ("gpu", JsValue::from_bool(g2.is_some())),
+                    ("passes", ((after.0 - before.0) as f64).into()),
+                    ("readbacks", ((after.1 - before.1) as f64).into()),
+                    ("ms", (t0.elapsed().as_secs_f64() * 1000.0).into()),
+                ]),
+                None,
+            );
+        }
+        send(r);
     });
+    effectcraft_engine::offload::run_request_async(&mut s, req, &send2).await;
+    SESSION.with(|c| *c.borrow_mut() = Some(s));
 }