aleleba/lightcraft
1
0
Fork 0
mirror of https://github.com/storytold/lightcraft.git synced 2026-10-24 10:09:01 -06:00
Code Issues Packages Projects Releases Wiki Activity

M0.12: web persistence — library, edits and imported originals survive a reload

Browse Source 
The catalog Store on the web is an in-memory mirror (apps/lightcraft-web/src/files.rs) loaded
from browser storage at start-up and flushed in the background in modification order (so a
snapshot always lands before the log reset that follows it). Storage is OPFS when writable from
the main thread (createWritable), else IndexedDB (?store=idb forces it); ?reset clears it.
Imported originals are stored as originals/<content hash> and referenced as
web/<hash>/<name>; the main thread keeps a bounded memory cache and loads misses (and the
active photo) asynchronously. view.json and ui.json are saved every second when changed.

Engine: Session::open_library_in(LibraryStores) opens a library from any Store (presets, view
and prefs go through it too); Session::save_view; Copy imports into a non-disk library add.
JS automation API: lightcraft.command(id, paramsJson) (+ web.stats).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas
2026-10-01 08:18:01 -04:00
co-authored by Claude Opus 5.5
parent 8c49450872
commit baa30ca590
11 changed files with 1147 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+2
View File
@@ -2229,9 +2229,11 @@ dependencies = [
"js-sys",
"lightcraft-codecs",
"lightcraft-engine",
"lightcraft-preview",
"lightcraft-raster",
"lightcraft-ui-egui",
"log",
"serde",
"serde_json",
"wasm-bindgen",
"wasm-bindgen-futures",
apps/lightcraft-web/Cargo.toml
+28
View File
@@ -17,6 +17,8 @@ lightcraft-engine = { workspace = true }
lightcraft-ui-egui = { workspace = true }
lightcraft-codecs = { workspace = true }
lightcraft-raster = { workspace = true }
lightcraft-preview = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
log = { workspace = true }
@@ -29,16 +31,42 @@ js-sys = { workspace = true }
web-sys = { workspace = true, features = [
"Blob",
"BlobPropertyBag",
"DedicatedWorkerGlobalScope",
"Document",
"DomStringList",
"Element",
"EventTarget",
"File",
"FileList",
"FileSystemCreateWritableOptions",
"FileSystemDirectoryHandle",
"FileSystemFileHandle",
"FileSystemGetDirectoryOptions",
"FileSystemGetFileOptions",
"FileSystemRemoveOptions",
"FileSystemWritableFileStream",
"HtmlAnchorElement",
"HtmlCanvasElement",
"HtmlElement",
"HtmlInputElement",
"IdbDatabase",
"IdbFactory",
"IdbKeyRange",
"IdbObjectStore",
"IdbOpenDbRequest",
"IdbRequest",
"IdbTransaction",
"IdbTransactionMode",
"IdbVersionChangeEvent",
"Location",
"MessageEvent",
"Navigator",
"Performance",
"StorageManager",
"Url",
"Window",
"Worker",
"WorkerOptions",
"WorkerType",
"WritableStream",
] }
apps/lightcraft-web/index.html
+10 -2
View File
@@ -22,8 +22,16 @@
<div id="lightcraft_loading">Loading LightCraft…</div>
<div id="lightcraft_status" aria-live="polite"></div>
<script type="module">
import init from "./lightcraft_web.js";
init().catch((e) => {
import init, { start, command } from "./lightcraft_web.js";
(async () => {
// Compile once; the render workers (worker.js) instantiate the same compiled module.
const module = await WebAssembly.compileStreaming(fetch("./lightcraft_web_bg.wasm"));
globalThis.lightcraftModule = module;
await init({ module_or_path: module });
// automation: `await lightcraft.command("library.info", "{}")` → JSON text
globalThis.lightcraft = { command };
start();
})().catch((e) => {
console.error(e);
const el = document.getElementById("lightcraft_loading");
if (el) el.textContent = "LightCraft failed to load: " + e;
apps/lightcraft-web/src/backend.rs
+324
View File
@@ -0,0 +1,324 @@
//! Browser storage, asynchronous: OPFS (Origin Private File System) when the page can write it
//! from the main thread (`FileSystemFileHandle.createWritable`), else IndexedDB. Used from the
//! window and from render workers alike. Keys are `/`-separated paths (OPFS directories; plain
//! IndexedDB keys).
use js_sys::{Reflect, Uint8Array};
use wasm_bindgen::prelude::*;
use wasm_bindgen::{JsCast, JsValue};
use wasm_bindgen_futures::JsFuture;
use web_sys::{FileSystemDirectoryHandle, FileSystemFileHandle, FileSystemGetDirectoryOptions, FileSystemGetFileOptions, IdbDatabase};
const IDB_NAME: &str = "lightcraft";
const IDB_STORE: &str = "files";
#[derive(Clone)]
pub enum Backend {
Opfs(FileSystemDirectoryHandle),
Idb(IdbDatabase),
}
fn err(e: JsValue) -> String {
e.as_string().or_else(|| e.dyn_ref::<js_sys::Error>().map(|e| String::from(e.message()))).unwrap_or_else(|| format!("{e:?}"))
}
/// `navigator.storage` in a window or a worker.
fn storage_manager() -> Option<web_sys::StorageManager> {
let g = js_sys::global();
let nav = Reflect::get(&g, &"navigator".into()).ok()?;
let storage = Reflect::get(&nav, &"storage".into()).ok()?;
(!storage.is_undefined()).then(|| storage.unchecked_into())
}
fn indexed_db() -> Option<web_sys::IdbFactory> {
let f = Reflect::get(&js_sys::global(), &"indexedDB".into()).ok()?;
(!f.is_undefined() && !f.is_null()).then(|| f.unchecked_into())
}
/// Can files be written through OPFS writable streams here?
fn opfs_writable() -> bool {
let g = js_sys::global();
Reflect::get(&g, &"FileSystemFileHandle".into())
.ok()
.filter(|c| !c.is_undefined())
.and_then(|c| Reflect::get(&c, &"prototype".into()).ok())
.and_then(|p| Reflect::get(&p, &"createWritable".into()).ok())
.is_some_and(|f| f.is_function())
}
/// Resolve an IndexedDB request.
async fn idb_req(req: &web_sys::IdbRequest) -> Result<JsValue, String> {
let r1 = req.clone();
let p = js_sys::Promise::new(&mut |resolve, reject| {
let r2 = r1.clone();
let ok = Closure::once_into_js(move || {
let _ = resolve.call1(&JsValue::NULL, &r2.result().unwrap_or(JsValue::UNDEFINED));
});
let fail = Closure::once_into_js(move |e: JsValue| {
let _ = reject.call1(&JsValue::NULL, &e);
});
r1.set_onsuccess(Some(ok.unchecked_ref()));
r1.set_onerror(Some(fail.unchecked_ref()));
});
JsFuture::from(p).await.map_err(err)
}
/// Wait for an IndexedDB transaction to commit.
async fn idb_done(tx: &web_sys::IdbTransaction) -> Result<(), String> {
let t1 = tx.clone();
let p = js_sys::Promise::new(&mut |resolve, reject| {
let ok = Closure::once_into_js(move || {
let _ = resolve.call0(&JsValue::NULL);
});
let rej = reject.clone();
let fail = Closure::once_into_js(move |e: JsValue| {
let _ = rej.call1(&JsValue::NULL, &e);
});
let abort = Closure::once_into_js(move |e: JsValue| {
let _ = reject.call1(&JsValue::NULL, &e);
});
t1.set_oncomplete(Some(ok.unchecked_ref()));
t1.set_onerror(Some(fail.unchecked_ref()));
t1.set_onabort(Some(abort.unchecked_ref()));
});
JsFuture::from(p).await.map(|_| ()).map_err(err)
}
impl Backend {
/// OPFS if writable from here (unless `prefer_idb`), else IndexedDB.
pub async fn open(prefer_idb: bool) -> Result<Backend, String> {
if !prefer_idb
&& opfs_writable()
&& let Some(sm) = storage_manager()
{
match JsFuture::from(sm.get_directory()).await {
Ok(root) => return Ok(Backend::Opfs(root.unchecked_into())),
Err(e) => log::warn!("OPFS unavailable ({}); using IndexedDB", err(e)),
}
}
Self::open_idb().await
}
async fn open_idb() -> Result<Backend, String> {
let f = indexed_db().ok_or("no IndexedDB")?;
let req = f.open_with_u32(IDB_NAME, 1).map_err(err)?;
let upgrade = Closure::<dyn FnMut(web_sys::IdbVersionChangeEvent)>::new(move |e: web_sys::IdbVersionChangeEvent| {
if let Some(db) = e.target().and_then(|t| t.dyn_into::<web_sys::IdbOpenDbRequest>().ok()).and_then(|r| r.result().ok()) {
let db: IdbDatabase = db.unchecked_into();
if !db.object_store_names().contains(IDB_STORE) {
let _ = db.create_object_store(IDB_STORE);
}
}
});
req.set_onupgradeneeded(Some(upgrade.as_ref().unchecked_ref()));
let db = idb_req(&req).await?;
drop(upgrade);
Ok(Backend::Idb(db.unchecked_into()))
}
/// "opfs" or "idb" (workers open the same kind).
pub fn kind(&self) -> &'static str {
match self {
Backend::Opfs(_) => "opfs",
Backend::Idb(_) => "idb",
}
}
/// The directory holding `path`'s last component (created when `create`).
async fn parent(root: &FileSystemDirectoryHandle, path: &str, create: bool) -> Result<Option<(FileSystemDirectoryHandle, String)>, String> {
let mut parts: Vec<&str> = path.split('/').filter(|p| !p.is_empty()).collect();
let name = parts.pop().ok_or("empty path")?.to_string();
let mut dir = root.clone();
for p in parts {
let opts = FileSystemGetDirectoryOptions::new();
opts.set_create(create);
match JsFuture::from(dir.get_directory_handle_with_options(p, &opts)).await {
Ok(d) => dir = d.unchecked_into(),
Err(e) if !create => {
let _ = e;
return Ok(None);
}
Err(e) => return Err(err(e)),
}
}
Ok(Some((dir, name)))
}
async fn opfs_file(root: &FileSystemDirectoryHandle, path: &str, create: bool) -> Result<Option<FileSystemFileHandle>, String> {
let Some((dir, name)) = Self::parent(root, path, create).await? else { return Ok(None) };
let opts = FileSystemGetFileOptions::new();
opts.set_create(create);
match JsFuture::from(dir.get_file_handle_with_options(&name, &opts)).await {
Ok(h) => Ok(Some(h.unchecked_into())),
Err(_) if !create => Ok(None),
Err(e) => Err(err(e)),
}
}
fn idb_store(db: &IdbDatabase, write: bool) -> Result<(web_sys::IdbTransaction, web_sys::IdbObjectStore), String> {
let mode = if write { web_sys::IdbTransactionMode::Readwrite } else { web_sys::IdbTransactionMode::Readonly };
let tx = db.transaction_with_str_and_mode(IDB_STORE, mode).map_err(err)?;
let store = tx.object_store(IDB_STORE).map_err(err)?;
Ok((tx, store))
}
/// A whole file, or `None` if it doesn't exist.
pub async fn read(&self, path: &str) -> Result<Option<Vec<u8>>, String> {
match self {
Backend::Opfs(root) => {
let Some(h) = Self::opfs_file(root, path, false).await? else { return Ok(None) };
let file: web_sys::File = JsFuture::from(h.get_file()).await.map_err(err)?.unchecked_into();
let buf = JsFuture::from(file.array_buffer()).await.map_err(err)?;
Ok(Some(Uint8Array::new(&buf).to_vec()))
}
Backend::Idb(db) => {
let (_tx, store) = Self::idb_store(db, false)?;
let v = idb_req(&store.get(&path.into()).map_err(err)?).await?;
if v.is_undefined() || v.is_null() {
return Ok(None);
}
Ok(Some(Uint8Array::new(&v).to_vec()))
}
}
}
/// Replace a file atomically (OPFS: the writable's swap file is committed on close).
pub async fn write(&self, path: &str, data: &[u8]) -> Result<(), String> {
self.write_at(path, None, data).await
}
/// Write `data` at `offset` keeping the bytes before it (`None`: replace the file). Atomic.
pub async fn write_at(&self, path: &str, offset: Option<usize>, data: &[u8]) -> Result<(), String> {
match self {
Backend::Opfs(root) => {
let h = Self::opfs_file(root, path, true).await?.ok_or("can't create file")?;
let opts = web_sys::FileSystemCreateWritableOptions::new();
opts.set_keep_existing_data(offset.is_some());
let w: web_sys::FileSystemWritableFileStream =
JsFuture::from(h.create_writable_with_options(&opts)).await.map_err(err)?.unchecked_into();
let res = async {
if let Some(off) = offset {
JsFuture::from(w.truncate_with_f64(off as f64).map_err(err)?).await.map_err(err)?;
JsFuture::from(w.seek_with_f64(off as f64).map_err(err)?).await.map_err(err)?;
}
JsFuture::from(w.write_with_js_u8_array(&Uint8Array::from(data)).map_err(err)?).await.map_err(err)?;
Ok::<(), String>(())
}
.await;
match res {
Ok(()) => JsFuture::from(w.close()).await.map(|_| ()).map_err(err),
Err(e) => {
let _ = JsFuture::from(w.abort()).await;
Err(e)
}
}
}
Backend::Idb(db) => {
let (tx, store) = Self::idb_store(db, true)?;
let value = match offset {
// read-modify-write inside one transaction
Some(off) => {
let old = idb_req(&store.get(&path.into()).map_err(err)?).await?;
let mut v = if old.is_undefined() || old.is_null() { Vec::new() } else { Uint8Array::new(&old).to_vec() };
v.truncate(off);
v.extend_from_slice(data);
Uint8Array::from(&v[..])
}
None => Uint8Array::from(data),
};
store.put_with_key(&value, &path.into()).map_err(err)?;
idb_done(&tx).await
}
}
}
#[allow(dead_code)] // thumbnail cache pruning (render workers)
pub async fn remove(&self, path: &str) -> Result<(), String> {
match self {
Backend::Opfs(root) => {
let Some((dir, name)) = Self::parent(root, path, false).await? else { return Ok(()) };
match JsFuture::from(dir.remove_entry(&name)).await {
Ok(_) => Ok(()),
Err(e) if Reflect::get(&e, &"name".into()).ok().and_then(|n| n.as_string()).as_deref() == Some("NotFoundError") => Ok(()),
Err(e) => Err(err(e)),
}
}
Backend::Idb(db) => {
let (tx, store) = Self::idb_store(db, true)?;
store.delete(&path.into()).map_err(err)?;
idb_done(&tx).await
}
}
}
/// Names of the files directly inside `dir`.
#[allow(dead_code)] // thumbnail cache pruning (render workers)
pub async fn list(&self, dir: &str) -> Result<Vec<String>, String> {
match self {
Backend::Opfs(root) => {
// `parent` of a child path is `dir` itself
let Some((handle, _)) = Self::parent(root, &format!("{dir}/-"), false).await? else { return Ok(Vec::new()) };
dir_keys(&handle).await
}
Backend::Idb(db) => {
let (_tx, store) = Self::idb_store(db, false)?;
let prefix = format!("{dir}/");
let range = web_sys::IdbKeyRange::bound(&prefix.as_str().into(), &format!("{prefix}\u{ffff}").into()).map_err(err)?;
let keys = idb_req(&store.get_all_keys_with_key(&range).map_err(err)?).await?;
let arr: js_sys::Array = keys.unchecked_into();
Ok(arr
.iter()
.filter_map(|k| k.as_string())
.filter_map(|k| k.strip_prefix(&prefix).filter(|r| !r.contains('/')).map(str::to_string))
.collect())
}
}
}
/// Delete everything (the "reset" escape hatch: `?reset` in the URL).
pub async fn clear(&self) -> Result<(), String> {
match self {
Backend::Opfs(root) => {
for name in dir_keys(root).await? {
let opts = web_sys::FileSystemRemoveOptions::new();
opts.set_recursive(true);
JsFuture::from(root.remove_entry_with_options(&name, &opts)).await.map_err(err)?;
}
Ok(())
}
Backend::Idb(db) => {
let (tx, store) = Self::idb_store(db, true)?;
store.clear().map_err(err)?;
idb_done(&tx).await
}
}
}
}
/// Names of the entries in an OPFS directory.
async fn dir_keys(dir: &FileSystemDirectoryHandle) -> Result<Vec<String>, String> {
let keys = dir.keys();
let mut out = Vec::new();
loop {
let next = JsFuture::from(keys.next().map_err(err)?).await.map_err(err)?;
if Reflect::get(&next, &"done".into()).ok().is_some_and(|d| d.is_truthy()) {
break;
}
if let Some(k) = Reflect::get(&next, &"value".into()).ok().and_then(|v| v.as_string()) {
out.push(k);
}
}
Ok(out)
}
/// Ask the browser not to evict our storage under pressure (best effort; fire and forget).
pub fn request_persistence() {
if let Some(sm) = storage_manager()
&& let Ok(p) = sm.persist()
{
wasm_bindgen_futures::spawn_local(async move {
let granted = JsFuture::from(p).await.ok().is_some_and(|v| v.is_truthy());
log::info!("lightcraft: persistent storage {}", if granted { "granted" } else { "not granted (best effort)" });
});
}
}
apps/lightcraft-web/src/bench.rs
+1 -1
View File
@@ -65,7 +65,7 @@ impl Bench {
if self.first_frame_ms.is_none() && self.frames > 1 {
self.first_frame_ms = Some(now - self.start_ms);
}
let idle = app.renderer.queued() == 0;
let idle = app.renderer.queued() == 0 && app.renderer.in_flight() == 0;
match self.phase {
Phase::Idle => {
let thumbs = app.renderer.textures.keys().filter(|s| matches!(s, Slot::Thumb(_))).count();
apps/lightcraft-web/src/files.rs
+255
View File
@@ -0,0 +1,255 @@
//! The library's small files (catalog journal, presets, view state, prefs) on the web.
//!
//! Browser storage is asynchronous on the main thread (OPFS writable streams, IndexedDB), but the
//! catalog [`Store`] is synchronous. So the files are mirrored in memory: they are read once at
//! start-up ([`Files::preload`]), every write changes the mirror immediately and marks the file
//! dirty, and the host flushes dirty files to storage in the background ([`Files::take_dirty`],
//! about once per frame while anything is dirty).
//!
//! Durability: a write is in browser storage a few milliseconds after the command that made it
//! (one flush), not before the command returns as on the desktop. Each flushed file is replaced
//! atomically (OPFS swap file / one IndexedDB transaction), and files are flushed in the order they
//! were last modified, so the journal's recovery rules still hold: e.g. a new `catalog.snap` always
//! lands before the `catalog.log` reset that follows it.
use std::collections::BTreeMap;
use std::io;
use std::sync::{Arc, Mutex};
use lightcraft_engine::catalog::Store;
/// What has to be written for a dirty file.
#[derive(Clone, Debug, PartialEq)]
pub enum FlushOp {
/// Replace the whole file.
Write { name: String, data: Vec<u8> },
/// Bytes appended since the last flush, starting at `offset` (the stored file's length).
Append { name: String, offset: usize, data: Vec<u8> },
}
impl FlushOp {
pub fn name(&self) -> &str {
match self {
FlushOp::Write { name, .. } | FlushOp::Append { name, .. } => name,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
enum Dirty {
Rewrite,
/// Only appended since the last flush; the stored file has `from` bytes.
Append {
from: usize,
},
}
#[derive(Default)]
struct Mirror {
files: BTreeMap<String, Vec<u8>>,
/// Dirty files, least recently modified first.
dirty: Vec<(String, Dirty)>,
}
impl Mirror {
fn mark(&mut self, name: &str, d: Dirty) {
let prev = self.dirty.iter().position(|(n, _)| n == name).map(|i| self.dirty.remove(i).1);
let d = match (prev, d) {
// appends after appends keep the original offset; anything else needs a rewrite
(Some(Dirty::Append { from }), Dirty::Append { .. }) => Dirty::Append { from },
(Some(Dirty::Rewrite), _) => Dirty::Rewrite,
(_, d) => d,
};
self.dirty.push((name.to_string(), d));
}
}
/// Shared in-memory mirror of the library files (cheap to clone).
#[derive(Clone, Default)]
pub struct Files(Arc<Mutex<Mirror>>);
impl Files {
fn lock(&self) -> std::sync::MutexGuard<'_, Mirror> {
self.0.lock().unwrap_or_else(|e| e.into_inner())
}
/// Set a file's stored content (start-up; not dirty).
pub fn preload(&self, name: &str, data: Vec<u8>) {
self.lock().files.insert(name.to_string(), data);
}
pub fn get(&self, name: &str) -> Option<Vec<u8>> {
self.lock().files.get(name).cloned()
}
/// Replace a file.
pub fn write(&self, name: &str, data: &[u8]) {
let mut m = self.lock();
m.files.insert(name.to_string(), data.to_vec());
m.mark(name, Dirty::Rewrite);
}
pub fn append(&self, name: &str, data: &[u8]) {
let mut m = self.lock();
let f = m.files.entry(name.to_string()).or_default();
let from = f.len();
f.extend_from_slice(data);
m.mark(name, Dirty::Append { from });
}
pub fn truncate(&self, name: &str, len: usize) {
let mut m = self.lock();
if let Some(f) = m.files.get_mut(name) {
f.truncate(len);
m.mark(name, Dirty::Rewrite);
}
}
pub fn is_dirty(&self) -> bool {
!self.lock().dirty.is_empty()
}
/// Everything to write, in modification order; the files count as clean afterwards (call
/// [`Files::failed`] for ops that couldn't be written).
pub fn take_dirty(&self) -> Vec<FlushOp> {
let mut m = self.lock();
let dirty = std::mem::take(&mut m.dirty);
dirty
.into_iter()
.map(|(name, d)| {
let data = m.files.get(&name).cloned().unwrap_or_default();
match d {
Dirty::Append { from } if from <= data.len() => FlushOp::Append { offset: from, data: data[from..].to_vec(), name },
_ => FlushOp::Write { name, data },
}
})
.collect()
}
/// A flush failed: write these files in full next time (keeps any newer modification order).
pub fn failed(&self, ops: &[FlushOp]) {
let mut m = self.lock();
for op in ops.iter().rev() {
if !m.dirty.iter().any(|(n, _)| n == op.name()) {
m.dirty.insert(0, (op.name().to_string(), Dirty::Rewrite));
} else if let Some(d) = m.dirty.iter_mut().find(|(n, _)| n == op.name()) {
d.1 = Dirty::Rewrite;
}
}
}
/// A catalog [`Store`] over these files.
pub fn store(&self) -> FilesStore {
FilesStore(self.clone())
}
}
/// [`Store`] implementation over [`Files`].
pub struct FilesStore(Files);
impl Store for FilesStore {
fn read(&mut self, name: &str) -> io::Result<Option<Vec<u8>>> {
Ok(self.0.get(name))
}
fn write_atomic(&mut self, name: &str, data: &[u8]) -> io::Result<()> {
self.0.write(name, data);
Ok(())
}
fn append(&mut self, name: &str, data: &[u8]) -> io::Result<()> {
self.0.append(name, data);
Ok(())
}
fn truncate(&mut self, name: &str, len: u64) -> io::Result<()> {
self.0.truncate(name, len as usize);
Ok(())
}
fn describe(&self) -> String {
"browser storage".into()
}
}
/// Names of the library files read at start-up.
pub const LIBRARY_FILES: [&str; 6] = ["catalog.snap", "catalog.log", "presets.json", "view.json", "prefs.json", "ui.json"];
#[cfg(test)]
mod tests {
use super::*;
use lightcraft_engine::Session;
use lightcraft_engine::library::LibraryStores;
use serde_json::json;
/// Apply flush ops to a "disk" map, as the backend would.
fn flush(files: &Files, disk: &mut BTreeMap<String, Vec<u8>>) {
for op in files.take_dirty() {
match op {
FlushOp::Write { name, data } => {
disk.insert(name, data);
}
FlushOp::Append { name, offset, data } => {
let f = disk.entry(name).or_default();
assert_eq!(f.len(), offset, "append offset matches the stored length");
f.extend_from_slice(&data);
}
}
}
}
#[test]
fn appends_coalesce_and_rewrites_win() {
let f = Files::default();
f.preload("log", b"ab".to_vec());
f.append("log", b"c");
f.append("log", b"d");
assert_eq!(f.take_dirty(), vec![FlushOp::Append { name: "log".into(), offset: 2, data: b"cd".to_vec() }]);
assert!(!f.is_dirty());
f.write("snap", b"S");
f.truncate("log", 1);
f.append("log", b"x");
let ops = f.take_dirty();
assert_eq!(
ops,
vec![FlushOp::Write { name: "snap".into(), data: b"S".to_vec() }, FlushOp::Write { name: "log".into(), data: b"ax".to_vec() }]
);
f.failed(&ops);
assert_eq!(f.take_dirty().len(), 2);
}
#[test]
fn modification_order_puts_snapshot_before_log_reset() {
let f = Files::default();
f.append("catalog.log", b"1\n");
f.write("catalog.snap", b"snap");
f.write("catalog.log", b"");
let names: Vec<_> = f.take_dirty().iter().map(|o| o.name().to_string()).collect();
assert_eq!(names, ["catalog.snap", "catalog.log"]);
}
#[test]
fn library_survives_reload_through_flushes() {
let mut disk = BTreeMap::new();
let open = |disk: &BTreeMap<String, Vec<u8>>| {
let files = Files::default();
for (k, v) in disk {
files.preload(k, v.clone());
}
let mut s = Session::new();
let stores = LibraryStores { dir: "browser".into(), catalog: Box::new(files.store()), files: Box::new(files.store()), on_disk: false };
s.open_library_in(stores, true).unwrap();
(s, files)
};
let (mut s, files) = open(&disk);
assert!(!s.catalog.is_empty(), "seeded with the demo photos");
flush(&files, &mut disk);
let id = s.selection.active.unwrap();
s.execute("develop.set", &json!({"control": "light.exposure", "value": 1.25})).unwrap();
s.execute("photo.rate", &json!({"rating": 4})).unwrap();
s.save_view();
flush(&files, &mut disk);
let (s2, _) = open(&disk);
let p = s2.catalog.photo(id).unwrap();
assert_eq!(p.rating, 4);
assert_eq!(p.develop.light.exposure, 1.25);
assert_eq!(s2.catalog.len(), s.catalog.len(), "not seeded twice");
assert_eq!(s2.selection.active, s.selection.active);
}
}
apps/lightcraft-web/src/lib.rs
+12 -8
View File
@@ -1,20 +1,24 @@
//! LightCraft in the browser: the same egui UI as the desktop app, compiled to WASM.
//!
//! Build with `cargo xtask web` (→ `target/web/`), then serve that folder over HTTP.
//! Differences from the desktop host:
//! - no filesystem: photos picked (File ▸ Add Photos…) or dropped onto the page are held in
//! memory ([`store::MemStore`]) and decoded from bytes;
//! - export downloads the file through the browser;
//! Build with `cargo xtask web` (→ `target/web/`), then serve that folder over HTTP (see
//! `docs/web.md`). Differences from the desktop host:
//! - the library lives in browser storage (OPFS, or IndexedDB where OPFS can't be written from the
//! main thread): catalog journal, presets, view state and prefs are mirrored in memory and
//! flushed in the background ([`files`]); imported originals are stored by content hash
//! ([`store`]); a new library starts with the procedural demo photos;
//! - renders run inline on the main thread (one job per frame) — no worker threads yet;
//! - no persistence yet (UI prefs and the catalog reset on reload);
//! - the procedural demo library is loaded at start so the page is never empty.
//! - export downloads the file through the browser.
//!
//! `?bench` in the URL runs a scripted first-paint / slider-latency measurement and logs it to the
//! console (see [`bench`]).
//! console (see [`bench`]); `?store=idb` forces the IndexedDB backend, `?store=memory` disables
//! persistence and `?reset` clears the stored library.
#![forbid(unsafe_code)]
pub mod bench;
pub mod files;
pub mod store;
#[cfg(target_arch = "wasm32")]
mod backend;
#[cfg(target_arch = "wasm32")]
mod web;
apps/lightcraft-web/src/store.rs
+147 -47
View File
@@ -1,6 +1,13 @@
//! In-memory "filesystem" for the web build: files picked or dropped in the browser are kept as
//! bytes under synthetic paths (`mem/<n>/<name>`), and the engine reads them through the same
//! [`FileLoader`]/[`FileProbe`] hooks the desktop app backs with `std::fs`.
//! Imported originals on the web. There is no filesystem: the bytes of every photo picked or
//! dropped in the browser are stored in browser storage as `originals/<content hash>`, and the
//! catalog refers to them by a synthetic path `web/<content hash>/<file name>` (so the decoders
//! still see the file's extension).
//!
//! The engine reads files synchronously ([`FileLoader`]/[`FileProbe`]), so the main thread keeps
//! the bytes it needs in a bounded memory cache ([`Originals`]). A miss (e.g. after a reload)
//! queues an asynchronous load from storage ([`Originals::take_wanted`]) and fails for now; the
//! host prefetches the active photo's original so commands that need its pixels (auto tone,
//! export) find it. Render workers read originals from storage themselves.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
@@ -8,37 +15,119 @@ use std::sync::{Arc, Mutex};
use lightcraft_engine::files::{load_bytes, probe_bytes};
use lightcraft_engine::media::{FileLoader, FileProbe};
#[derive(Default)]
struct Inner {
files: HashMap<String, Arc<[u8]>>,
/// Paths added since the last [`MemStore::take_pending`] (to import on the next frame).
pending: Vec<String>,
next: u64,
/// Prefix of the catalog paths of files kept in browser storage.
pub const PATH_PREFIX: &str = "web/";
/// Memory budget for original bytes on the main thread.
const MEM_BUDGET: usize = 768 << 20;
/// Catalog path for a stored original.
pub fn original_path(hash: &str, name: &str) -> String {
let name = name.rsplit(['/', '\\']).next().filter(|n| !n.is_empty()).unwrap_or("photo");
format!("{PATH_PREFIX}{hash}/{name}")
}
/// Shared, cloneable byte store. File reads in the browser are async, so additions arrive from
/// promise callbacks and are picked up by the app on its next frame.
#[derive(Clone, Default)]
pub struct MemStore(Arc<Mutex<Inner>>);
/// The content hash in a path made by [`original_path`].
pub fn hash_of_path(path: &str) -> Option<&str> {
let (hash, _) = path.strip_prefix(PATH_PREFIX)?.split_once('/')?;
(!hash.is_empty()).then_some(hash)
}
impl MemStore {
/// Storage key of an original.
pub fn storage_key(hash: &str) -> String {
format!("originals/{hash}")
}
/// Hash of a file's bytes (the same as the catalog's `content_hash`).
pub fn content_hash(bytes: &[u8]) -> String {
lightcraft_preview::hash_bytes(bytes).to_string()
}
#[derive(Default)]
struct Inner {
/// hash → (bytes, last use).
bytes: HashMap<String, (Arc<[u8]>, u64)>,
total: usize,
clock: u64,
/// Catalog paths stored since the last [`Originals::take_pending`] (to import on the next frame).
pending: Vec<String>,
/// Hashes to load from storage.
wanted: Vec<String>,
/// Hashes being loaded.
loading: std::collections::HashSet<String>,
}
/// Original bytes in memory, shared between the engine hooks and the host (cheap to clone).
#[derive(Clone, Default)]
pub struct Originals(Arc<Mutex<Inner>>);
impl Originals {
fn lock(&self) -> std::sync::MutexGuard<'_, Inner> {
self.0.lock().unwrap_or_else(|e| e.into_inner())
}
/// Store a file's bytes; returns its synthetic path and queues it for import.
pub fn add(&self, name: &str, bytes: Vec<u8>) -> String {
/// Keep `bytes` (hash `hash`) in memory, evicting the least recently used others if needed.
pub fn insert(&self, hash: &str, bytes: Arc<[u8]>) {
let mut g = self.lock();
g.next += 1;
let name = name.rsplit(['/', '\\']).next().filter(|n| !n.is_empty()).unwrap_or("photo");
let path = format!("mem/{}/{name}", g.next);
g.files.insert(path.clone(), bytes.into());
g.pending.push(path.clone());
g.clock += 1;
let clock = g.clock;
g.loading.remove(hash);
if let Some(old) = g.bytes.insert(hash.to_string(), (bytes.clone(), clock)) {
g.total -= old.0.len();
}
g.total += bytes.len();
while g.total > MEM_BUDGET && g.bytes.len() > 1 {
let Some(oldest) = g.bytes.iter().filter(|(h, _)| *h != hash).min_by_key(|(_, v)| v.1).map(|(h, _)| h.clone()) else { break };
if let Some(v) = g.bytes.remove(&oldest) {
g.total -= v.0.len();
}
}
}
/// A file was stored (bytes in storage and in memory): queue its path for import.
pub fn added(&self, name: &str, hash: &str, bytes: Arc<[u8]>) -> String {
self.insert(hash, bytes);
let path = original_path(hash, name);
self.lock().pending.push(path.clone());
path
}
/// Bytes for a catalog path, if in memory. A miss for a stored original queues its load.
pub fn get(&self, path: &str) -> Option<Arc<[u8]>> {
self.lock().files.get(path).cloned()
let hash = hash_of_path(path)?;
let mut g = self.lock();
g.clock += 1;
let clock = g.clock;
if let Some(v) = g.bytes.get_mut(hash) {
v.1 = clock;
return Some(v.0.clone());
}
if !g.loading.contains(hash) && !g.wanted.iter().any(|w| w == hash) {
g.wanted.push(hash.to_string());
}
None
}
pub fn contains(&self, path: &str) -> bool {
hash_of_path(path).is_some_and(|h| self.lock().bytes.contains_key(h))
}
/// Ask for the original behind `path` to be loaded into memory (no-op if it's there).
pub fn prefetch(&self, path: &str) {
let _ = self.get(path);
}
/// Hashes to load from storage (they count as loading until [`Originals::insert`] or
/// [`Originals::load_failed`]).
pub fn take_wanted(&self) -> Vec<String> {
let mut g = self.lock();
let w = std::mem::take(&mut g.wanted);
g.loading.extend(w.iter().cloned());
w
}
pub fn load_failed(&self, hash: &str) {
self.lock().loading.remove(hash);
}
/// Paths added since the last call.
@@ -46,24 +135,22 @@ impl MemStore {
std::mem::take(&mut self.lock().pending)
}
pub fn len(&self) -> usize {
self.lock().files.len()
/// (originals in memory, bytes).
pub fn usage(&self) -> (usize, usize) {
let g = self.lock();
(g.bytes.len(), g.total)
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
/// Engine hooks that decode/probe from this store.
/// Engine hooks that decode/probe from memory.
pub fn hooks(&self) -> (FileLoader, FileProbe) {
let s = self.clone();
let loader: FileLoader = Arc::new(move |path: &str, max_edge: usize| {
let bytes = s.get(path).ok_or_else(|| format!("{path}: not in memory"))?;
let bytes = s.get(path).ok_or_else(|| format!("{path}: the original is still loading from browser storage"))?;
load_bytes(&bytes, max_edge)
});
let s = self.clone();
let probe: FileProbe = Arc::new(move |path: &str| {
let bytes = s.get(path).ok_or_else(|| format!("{path}: not in memory"))?;
let bytes = s.get(path).ok_or_else(|| format!("{path}: not loaded"))?;
probe_bytes(path, &bytes)
});
(loader, probe)
@@ -94,41 +181,54 @@ pub fn mime_for(name: &str) -> &'static str {
}
}
#[cfg(test)]
pub(crate) fn png_bytes(w: usize, h: usize) -> Vec<u8> {
let bytes: Vec<u8> = (0..w * h).flat_map(|i| [(i * 7) as u8, 128, 200, 255]).collect();
let img = lightcraft_raster::Rgba8::from_bytes(w, h, &bytes).unwrap();
lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(&img), &lightcraft_codecs::EncodeMeta::default()).unwrap()
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
fn png_bytes(w: usize, h: usize) -> Vec<u8> {
let bytes: Vec<u8> = (0..w * h).flat_map(|i| [(i * 7) as u8, 128, 200, 255]).collect();
let img = lightcraft_raster::Rgba8::from_bytes(w, h, &bytes).unwrap();
lightcraft_codecs::encode_png(&lightcraft_codecs::EncodeImage::rgba8(&img), &lightcraft_codecs::EncodeMeta::default()).unwrap()
#[test]
fn paths() {
let p = original_path("abc123", "C:\\x\\a.png");
assert_eq!(p, "web/abc123/a.png");
assert_eq!(hash_of_path(&p), Some("abc123"));
assert_eq!(hash_of_path("mem/1/a.png"), None);
assert_eq!(storage_key("abc"), "originals/abc");
}
#[test]
fn add_take_get() {
let s = MemStore::default();
let a = s.add("C:\\x\\a.png", vec![1, 2]);
let b = s.add("a.png", vec![3]);
assert_ne!(a, b, "same name twice gets distinct paths");
assert!(a.ends_with("/a.png"));
assert_eq!(s.take_pending(), vec![a.clone(), b]);
assert!(s.take_pending().is_empty());
assert_eq!(&*s.get(&a).unwrap(), &[1, 2]);
assert_eq!(s.len(), 2);
fn miss_queues_a_load_once() {
let s = Originals::default();
assert!(s.get("web/h1/a.png").is_none());
assert!(s.get("web/h1/a.png").is_none());
assert_eq!(s.take_wanted(), vec!["h1".to_string()]);
assert!(s.get("web/h1/a.png").is_none(), "still loading");
assert!(s.take_wanted().is_empty());
s.insert("h1", Arc::from(vec![1u8, 2]));
assert_eq!(&*s.get("web/h1/b.jpg").unwrap(), &[1, 2]);
}
#[test]
fn import_and_render_from_memory() {
let store = MemStore::default();
let store = Originals::default();
let mut session = lightcraft_engine::Session::new();
store.install(&mut session);
let path = store.add("tiny.png", png_bytes(40, 30));
let bytes = png_bytes(40, 30);
let hash = content_hash(&bytes);
let path = store.added("tiny.png", &hash, bytes.into());
assert_eq!(store.take_pending(), vec![path.clone()]);
let r = session.execute("library.import", &json!({"paths": [path]})).unwrap();
let id = lightcraft_engine::catalog::PhotoId(r["imported"][0].as_u64().unwrap());
let p = session.catalog.photo(id).unwrap();
assert_eq!((p.width, p.height), (40, 30));
assert_eq!(p.file_name, "tiny.png");
assert_eq!(p.content_hash.as_deref(), Some(hash.as_str()), "storage key = catalog content hash");
let out = session.render_now(id, 20, 20).unwrap();
assert_eq!(out.image.width, 20);
}
apps/lightcraft-web/src/web.rs
+295 -32
View File
@@ -1,17 +1,32 @@
//! Browser host: starts eframe on `<canvas id="lightcraft_canvas">`, wires the in-memory store to
//! the file picker and drag-and-drop, and turns exports into downloads.
//! Browser host: opens the library in browser storage, starts eframe on
//! `<canvas id="lightcraft_canvas">`, wires the file picker and drag-and-drop, and turns exports
//! into downloads.
use std::cell::Cell;
use std::rc::Rc;
use std::sync::Arc;
use lightcraft_engine::Session;
use lightcraft_ui_egui::{LightcraftApp, Services};
use lightcraft_engine::catalog::Source;
use lightcraft_engine::library::LibraryStores;
use lightcraft_ui_egui::{LightcraftApp, Services, UiState};
use serde_json::json;
use wasm_bindgen::JsCast;
use wasm_bindgen::prelude::*;
use crate::backend::Backend;
use crate::bench::Bench;
use crate::store::{MemStore, download_name};
use crate::files::{Files, FlushOp, LIBRARY_FILES};
use crate::store::{Originals, content_hash, download_name, storage_key};
const ACCEPT: &str = ".jpg,.jpeg,.png,.tif,.tiff,.webp,.dng,.cr2,.nef,.arw,.psd,.jxl,.gif,.bmp";
/// Where the library files live in browser storage.
const LIBRARY_DIR: &str = "library";
/// How often view state and UI prefs are saved (ms).
const SAVE_EVERY_MS: f64 = 1000.0;
fn window() -> web_sys::Window {
web_sys::window().expect("no window")
}
@@ -27,19 +42,62 @@ fn set_status(text: &str) {
}
}
/// Read a browser `File` into the store.
async fn read_file(store: MemStore, file: web_sys::File, ctx: egui::Context) {
match wasm_bindgen_futures::JsFuture::from(file.array_buffer()).await {
Ok(buf) => {
store.add(&file.name(), js_sys::Uint8Array::new(&buf).to_vec());
ctx.request_repaint();
fn library_key(name: &str) -> String {
format!("{LIBRARY_DIR}/{name}")
}
/// URL options (`?bench&store=idb&reset`).
struct Options {
bench: bool,
/// "opfs" (default: OPFS, falling back to IndexedDB), "idb" or "memory".
store: String,
reset: bool,
}
impl Options {
fn from_url() -> Options {
let q = window().location().search().unwrap_or_default();
let params: Vec<(String, String)> = q
.trim_start_matches('?')
.split('&')
.filter(|p| !p.is_empty())
.map(|p| match p.split_once('=') {
Some((k, v)) => (k.to_string(), v.to_string()),
None => (p.to_string(), String::new()),
})
.collect();
let get = |k: &str| params.iter().find(|(n, _)| n == k).map(|(_, v)| v.clone());
Options {
bench: get("bench").is_some(),
store: get("store").filter(|s| s == "idb" || s == "memory").unwrap_or_else(|| "opfs".into()),
reset: get("reset").is_some(),
}
}
}
/// Store a picked/dropped file (bytes into browser storage, then queue its import).
async fn store_file(originals: Originals, backend: Option<Backend>, name: String, bytes: Vec<u8>, ctx: egui::Context) {
let hash = content_hash(&bytes);
if let Some(b) = &backend
&& let Err(e) = b.write(&storage_key(&hash), &bytes).await
{
// still importable for this session; it won't survive a reload
log::error!("storing {name} in browser storage failed: {e}");
}
originals.added(&name, &hash, Arc::from(bytes));
ctx.request_repaint();
}
/// Read a browser `File` into storage.
async fn read_file(originals: Originals, backend: Option<Backend>, file: web_sys::File, ctx: egui::Context) {
match wasm_bindgen_futures::JsFuture::from(file.array_buffer()).await {
Ok(buf) => store_file(originals, backend, file.name(), js_sys::Uint8Array::new(&buf).to_vec(), ctx).await,
Err(e) => log::warn!("reading {}: {e:?}", file.name()),
}
}
/// Show the browser's open dialog; picked files land in the store asynchronously.
fn open_picker(store: MemStore, ctx: egui::Context) {
/// Show the browser's open dialog; picked files are imported asynchronously.
fn open_picker(originals: Originals, backend: Option<Backend>, ctx: egui::Context) {
let Some(doc) = window().document() else { return };
let Ok(input) = doc.create_element("input").map(|e| e.unchecked_into::<web_sys::HtmlInputElement>()) else { return };
input.set_type("file");
@@ -50,7 +108,7 @@ fn open_picker(store: MemStore, ctx: egui::Context) {
if let Some(files) = inp.files() {
for i in 0..files.length() {
if let Some(f) = files.get(i) {
wasm_bindgen_futures::spawn_local(read_file(store.clone(), f, ctx.clone()));
wasm_bindgen_futures::spawn_local(read_file(originals.clone(), backend.clone(), f, ctx.clone()));
}
}
}
@@ -83,10 +141,10 @@ fn download(path: &str, bytes: &[u8]) -> Result<(), String> {
Ok(())
}
fn services(store: MemStore, ctx: egui::Context) -> Services {
fn services(originals: Originals, backend: Option<Backend>, ctx: egui::Context) -> Services {
Services {
pick_files: Some(Box::new(move || {
open_picker(store.clone(), ctx.clone());
open_picker(originals.clone(), backend.clone(), ctx.clone());
Vec::new() // files arrive asynchronously and are imported on a later frame
})),
// Preset files: browser pickers are asynchronous; not wired on the web yet.
@@ -99,44 +157,145 @@ fn services(store: MemStore, ctx: egui::Context) -> Services {
}
}
/// Write dirty library files to storage, in order, until none are left.
async fn flush(files: Files, backend: Backend, flushing: Rc<Cell<bool>>) {
'outer: loop {
let ops = files.take_dirty();
if ops.is_empty() {
break;
}
for (i, op) in ops.iter().enumerate() {
let r = match op {
FlushOp::Write { name, data } => backend.write(&library_key(name), data).await,
FlushOp::Append { name, offset, data } => backend.write_at(&library_key(name), Some(*offset), data).await,
};
if let Err(e) = r {
log::error!("saving {} failed: {e}", op.name());
files.failed(&ops[i..]);
break 'outer;
}
}
}
flushing.set(false);
}
/// Everything loaded before the UI starts.
struct Boot {
opts: Options,
backend: Option<Backend>,
files: Files,
}
async fn boot(opts: Options) -> Boot {
let t0 = perf_now();
let backend = if opts.store == "memory" {
None
} else {
match Backend::open(opts.store == "idb").await {
Ok(b) => Some(b),
Err(e) => {
log::error!("no browser storage ({e}); this session won't be saved");
None
}
}
};
let files = Files::default();
if let Some(b) = &backend {
if opts.reset {
match b.clear().await {
Ok(()) => log::info!("lightcraft: storage cleared (?reset)"),
Err(e) => log::error!("clearing storage: {e}"),
}
}
for name in LIBRARY_FILES {
match b.read(&library_key(name)).await {
Ok(Some(data)) => files.preload(name, data),
Ok(None) => {}
Err(e) => log::error!("reading {name}: {e}"),
}
}
crate::backend::request_persistence();
}
log::info!("lightcraft: storage {} opened in {:.0} ms", backend.as_ref().map_or("memory", |b| b.kind()), perf_now() - t0);
Boot { opts, backend, files }
}
struct WebApp {
app: LightcraftApp,
store: MemStore,
originals: Originals,
files: Files,
backend: Option<Backend>,
flushing: Rc<Cell<bool>>,
bench: Option<Bench>,
first_frame_logged: bool,
last_save: f64,
ui_written: Vec<u8>,
}
impl WebApp {
fn new(cc: &eframe::CreationContext<'_>, bench: bool) -> Self {
let store = MemStore::default();
fn new(cc: &eframe::CreationContext<'_>, boot: Boot) -> Self {
let Boot { opts, backend, files } = boot;
let originals = Originals::default();
let t = perf_now();
let mut session = Session::with_demo();
log::info!("lightcraft: wasm started at {:.0} ms; demo library built in {:.0} ms", t, perf_now() - t);
store.install(&mut session);
let mut session = Session::new();
originals.install(&mut session);
let stores = LibraryStores {
dir: format!("browser:{}/{LIBRARY_DIR}", backend.as_ref().map_or("memory", |b| b.kind())).into(),
catalog: Box::new(files.store()),
files: Box::new(files.store()),
on_disk: false,
};
match session.open_library_in(stores, true).cloned() {
Ok(r) => log::info!(
"lightcraft: library {} in {:.0} ms ({} photos; snapshot seq {}, {} ops replayed)",
if r.created { "created" } else { "loaded" },
perf_now() - t,
session.catalog.len(),
r.snapshot_seq,
r.replayed
),
Err(e) => {
log::error!("opening the library failed: {e}; starting a temporary one");
session = Session::with_demo();
originals.install(&mut session);
}
}
// previews are large (≤ 2560 px, f32): keep few in a 32-bit address space
session.media.preview_capacity = 3;
let mut app = LightcraftApp::new(session, services(store.clone(), cc.egui_ctx.clone()));
let mut app = LightcraftApp::new(session, services(originals.clone(), backend.clone(), cc.egui_ctx.clone()));
let ui_written = files.get("ui.json").unwrap_or_default();
if let Ok(ui) = serde_json::from_slice::<UiState>(&ui_written) {
app.ui = ui;
}
app.ui = app.ui.sanitized();
CTX.with(|c| *c.borrow_mut() = Some(cc.egui_ctx.clone()));
let origin = lightcraft_ui_egui::now_ms() - perf_now();
WebApp { app, store, bench: bench.then(|| Bench::new(origin)), first_frame_logged: false }
WebApp {
app,
originals,
files,
backend,
flushing: Rc::new(Cell::new(false)),
bench: opts.bench.then(|| Bench::new(origin)),
first_frame_logged: false,
last_save: 0.0,
ui_written,
}
}
fn import_dropped(&mut self, ctx: &egui::Context) {
let dropped = ctx.input(|i| i.raw.dropped_files.clone());
for f in dropped {
let (store, ctx) = (self.store.clone(), ctx.clone());
let (originals, backend, ctx) = (self.originals.clone(), self.backend.clone(), ctx.clone());
wasm_bindgen_futures::spawn_local(async move {
let name = f.path().to_string_lossy().to_string();
match f.bytes_async().await {
Ok(bytes) => {
store.add(&name, bytes);
ctx.request_repaint();
}
Ok(bytes) => store_file(originals, backend, name, bytes, ctx).await,
Err(e) => log::warn!("reading dropped {name}: {e}"),
}
});
}
let paths = self.store.take_pending();
let paths = self.originals.take_pending();
if !paths.is_empty() {
let n = paths.len();
match self.app.run("library.import", json!({"paths": paths})) {
@@ -145,12 +304,114 @@ impl WebApp {
}
}
}
/// Load originals the main thread asked for (and the active photo's, ahead of time).
fn load_originals(&mut self, ctx: &egui::Context) {
if let Some(id) = self.app.session.selection.active
&& let Some(Source::File { path }) = self.app.session.catalog.photo(id).map(|p| &p.source)
&& !self.originals.contains(path)
{
self.originals.prefetch(path);
}
let Some(b) = &self.backend else { return };
for hash in self.originals.take_wanted() {
let (b, originals, ctx) = (b.clone(), self.originals.clone(), ctx.clone());
wasm_bindgen_futures::spawn_local(async move {
match b.read(&storage_key(&hash)).await {
Ok(Some(bytes)) => {
originals.insert(&hash, Arc::from(bytes));
ctx.request_repaint();
}
Ok(None) => {
log::warn!("original {hash} is not in browser storage");
originals.load_failed(&hash);
}
Err(e) => {
log::warn!("loading original {hash}: {e}");
originals.load_failed(&hash);
}
}
});
}
}
/// Save view state and UI prefs now and then; flush dirty files.
fn save(&mut self) {
let now = perf_now();
if now - self.last_save >= SAVE_EVERY_MS {
self.last_save = now;
self.app.session.save_view();
if let Ok(ui) = serde_json::to_vec_pretty(&self.app.ui)
&& ui != self.ui_written
{
self.files.write("ui.json", &ui);
self.ui_written = ui;
}
}
if let Some(b) = &self.backend
&& self.files.is_dirty()
&& !self.flushing.get()
{
self.flushing.set(true);
wasm_bindgen_futures::spawn_local(flush(self.files.clone(), b.clone(), self.flushing.clone()));
}
}
}
type Reply = (js_sys::Function, js_sys::Function);
thread_local! {
/// Commands from JavaScript ([`command`]), run on the next frame.
static INBOX: std::cell::RefCell<Vec<(String, String, Reply)>> = const { std::cell::RefCell::new(Vec::new()) };
static CTX: std::cell::RefCell<Option<egui::Context>> = const { std::cell::RefCell::new(None) };
}
/// Run a command by id from JavaScript (automation, tests): `await command("library.info", "{}")`
/// resolves to the result as JSON text, or rejects with the error. Besides every engine/UI
/// command, `web.stats` reports the browser host's state (storage, originals in memory).
#[wasm_bindgen]
pub fn command(id: String, params: String) -> js_sys::Promise {
js_sys::Promise::new(&mut |resolve, reject| {
INBOX.with(|q| q.borrow_mut().push((id.clone(), params.clone(), (resolve, reject))));
CTX.with(|c| c.borrow().as_ref().map(|c| c.request_repaint()));
})
}
impl WebApp {
fn web_stats(&self) -> serde_json::Value {
let (orig_n, orig_bytes) = self.originals.usage();
json!({
"storage": self.backend.as_ref().map_or("memory", |b| b.kind()),
"dirty": self.files.is_dirty(),
"flushing": self.flushing.get(),
"originalsInMemory": orig_n,
"originalBytesInMemory": orig_bytes,
"renderQueue": self.app.renderer.queued(),
"rendersInFlight": self.app.renderer.in_flight(),
"thumbTextures": self.app.renderer.thumb_textures(),
})
}
fn run_inbox(&mut self) {
let inbox = INBOX.with(|q| std::mem::take(&mut *q.borrow_mut()));
for (id, params, (resolve, reject)) in inbox {
let params: serde_json::Value = serde_json::from_str(&params).unwrap_or(json!({}));
let r = if id == "web.stats" { Ok(self.web_stats()) } else { self.app.run(&id, params) };
let _ = match r {
Ok(v) => resolve.call1(&JsValue::NULL, &v.to_string().into()),
Err(e) => reject.call1(&JsValue::NULL, &e.into()),
};
}
}
}
impl eframe::App for WebApp {
fn logic(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) {
self.run_inbox();
self.import_dropped(ctx);
self.load_originals(ctx);
self.app.logic(ctx);
self.save();
if let Some(b) = self.bench.as_mut()
&& let Some(report) = b.step(&mut self.app)
{
@@ -176,11 +437,12 @@ impl eframe::App for WebApp {
}
}
#[wasm_bindgen(start)]
/// Start the app (called by `index.html` after instantiating the module).
#[wasm_bindgen]
pub fn start() {
eframe::WebLogger::init(log::LevelFilter::Info).ok();
log::info!("lightcraft: wasm instantiated at {:.0} ms", perf_now());
let bench = window().location().search().is_ok_and(|q| q.contains("bench"));
let opts = Options::from_url();
wasm_bindgen_futures::spawn_local(async move {
let Some(canvas) =
window().document().and_then(|d| d.get_element_by_id("lightcraft_canvas")).and_then(|e| e.dyn_into::<web_sys::HtmlCanvasElement>().ok())
@@ -188,8 +450,9 @@ pub fn start() {
log::error!("missing <canvas id=\"lightcraft_canvas\">");
return;
};
let boot = boot(opts).await;
let runner = eframe::WebRunner::new();
let r = runner.start(canvas, eframe::WebOptions::default(), Box::new(move |cc| Ok(Box::new(WebApp::new(cc, bench))))).await;
let r = runner.start(canvas, eframe::WebOptions::default(), Box::new(move |cc| Ok(Box::new(WebApp::new(cc, boot))))).await;
match r {
Ok(()) => {
if let Some(el) = window().document().and_then(|d| d.get_element_by_id("lightcraft_loading")) {
crates/engine/src/import.rs
+4 -1
View File
@@ -152,7 +152,10 @@ fn copy_into_library(lib: &Path, src: &str, date: &str) -> Result<String, String
/// Import files/folders. See the module docs.
pub fn import(s: &mut Session, paths: &[String], mode: ImportMode) -> crate::Result<ImportReport> {
let lib_dir = s.library.as_ref().map(|l| l.dir.clone());
// Only a library on disk has an `Originals/` folder. The browser build keeps the bytes of every
// added file in its own storage already, so "copy" there means "add".
let mode = if mode == ImportMode::Copy && s.library.as_ref().is_some_and(|l| !l.on_disk) { ImportMode::Add } else { mode };
let lib_dir = s.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.clone());
if mode == ImportMode::Copy && lib_dir.is_none() {
return Err(crate::EngineError::Other("copying into the library needs an open library".into()));
}
crates/engine/src/library.rs
+69 -22
View File
@@ -10,6 +10,9 @@
//! Originals/ (photos imported with "copy into library")
//! ```
//!
//! The files live in [`Store`]s: a directory ([`FsStore`]) natively, or any other implementation
//! via [`Session::open_library_in`] (the browser build keeps them in OPFS or IndexedDB).
//!
//! Every top-level [`Session::execute`] persists the ops it produced (fsynced) before returning,
//! so a crash loses at most the command in flight. The log is compacted into a snapshot when it
//! grows (see [`lightcraft_catalog::SnapshotPolicy`]) and on [`Session::close_library`].
@@ -36,13 +39,32 @@ pub fn default_dir() -> Option<PathBuf> {
}
pub struct Library {
/// The library directory, or a descriptive pseudo-path when the library isn't on disk.
pub dir: PathBuf,
/// The library is a real directory: photos can be copied into `Originals/` and thumbnails are
/// cached in `thumbs/`.
pub on_disk: bool,
journal: Journal,
/// presets.json, view.json, prefs.json.
files: Box<dyn Store>,
/// What happened when the catalog was loaded.
pub report: LoadReport,
/// Last persistence error (shown by the UI; ops stay pending and are retried).
pub last_error: Option<String>,
presets_written: String,
view_written: Vec<u8>,
}
/// Where a library's files live (see [`Session::open_library_in`]).
pub struct LibraryStores {
/// Shown as the library's location (`library.info`).
pub dir: PathBuf,
/// The catalog journal (`catalog.snap`, `catalog.log`).
pub catalog: Box<dyn Store>,
/// presets.json, view.json, prefs.json.
pub files: Box<dyn Store>,
/// `dir` is a real directory (enables `Originals/` copies and the `thumbs/` disk cache).
pub on_disk: bool,
}
#[derive(Default, Serialize, Deserialize)]
@@ -87,8 +109,8 @@ fn presets_json(presets: &[Preset]) -> String {
serde_json::to_string_pretty(&f).unwrap_or_default()
}
fn write_atomic(dir: &Path, name: &str, data: &[u8]) -> std::io::Result<()> {
FsStore::open(dir)?.write_atomic(name, data)
fn read_json<T: serde::de::DeserializeOwned>(store: &mut dyn Store, name: &str) -> Option<T> {
store.read(name).ok().flatten().and_then(|b| serde_json::from_slice(&b).ok())
}
impl Session {
@@ -96,8 +118,16 @@ impl Session {
/// `seed_demo`, a newly created library starts with the procedural demo photos.
pub fn open_library(&mut self, dir: impl AsRef<Path>, seed_demo: bool) -> Result<&LoadReport> {
let dir = dir.as_ref().to_path_buf();
let store = FsStore::open(&dir).map_err(|e| EngineError::Other(format!("can't open library {}: {e}", dir.display())))?;
let (mut journal, catalog, report) = Journal::open(Box::new(store))?;
let open = || FsStore::open(&dir).map_err(|e| EngineError::Other(format!("can't open library {}: {e}", dir.display())));
let stores = LibraryStores { catalog: Box::new(open()?), files: Box::new(open()?), on_disk: true, dir: dir.clone() };
self.open_library_in(stores, seed_demo)
}
/// Open (or create) a library whose files live in `stores` (e.g. browser storage), replacing
/// this session's catalog.
pub fn open_library_in(&mut self, stores: LibraryStores, seed_demo: bool) -> Result<&LoadReport> {
let LibraryStores { dir, catalog, mut files, on_disk } = stores;
let (mut journal, catalog, report) = Journal::open(catalog)?;
self.catalog = catalog;
self.undo.clear();
self.redo.clear();
@@ -110,9 +140,7 @@ impl Session {
journal.snapshot(&self.catalog)?;
}
// presets
if let Ok(bytes) = std::fs::read(dir.join("presets.json"))
&& let Ok(f) = serde_json::from_slice::<PresetsFile>(&bytes)
{
if let Some(f) = read_json::<PresetsFile>(files.as_mut(), "presets.json") {
for p in &mut self.presets {
p.favorite = p.builtin && f.favorites.contains(&p.id) || (!p.builtin && p.favorite);
}
@@ -123,12 +151,9 @@ impl Session {
}
}
// preferences
self.xmp =
std::fs::read(dir.join("prefs.json")).ok().and_then(|b| serde_json::from_slice::<PrefsFile>(&b).ok()).map(|p| p.xmp).unwrap_or_default();
self.xmp = read_json::<PrefsFile>(files.as_mut(), "prefs.json").map(|p| p.xmp).unwrap_or_default();
// view state
if let Ok(bytes) = std::fs::read(dir.join("view.json"))
&& let Ok(v) = serde_json::from_slice::<ViewFile>(&bytes)
{
if let Some(v) = read_json::<ViewFile>(files.as_mut(), "view.json") {
self.source = v.source;
self.filter = v.filter;
self.sort = v.sort;
@@ -142,8 +167,11 @@ impl Session {
self.selection = Selection::single(*first);
}
let presets_written = presets_json(&self.presets);
self.media.attach_disk_cache(&dir.join("thumbs"));
self.library = Some(Library { dir, journal, report, last_error: None, presets_written });
if on_disk {
self.media.attach_disk_cache(&dir.join("thumbs"));
}
let view_written = self.view_json();
self.library = Some(Library { dir, on_disk, journal, files, report, last_error: None, presets_written, view_written });
Ok(&self.library.as_ref().expect("just set").report)
}
@@ -166,7 +194,7 @@ impl Session {
}
let presets = presets_json(&self.presets);
if presets != lib.presets_written {
if let Err(e) = write_atomic(&lib.dir, "presets.json", presets.as_bytes()) {
if let Err(e) = lib.files.write_atomic("presets.json", presets.as_bytes()) {
log::error!("library: presets: {e}");
} else {
lib.presets_written = presets;
@@ -190,20 +218,39 @@ impl Session {
}
let _ = self.end_interaction();
self.persist()?;
let view = ViewFile { source: self.source, filter: self.filter.clone(), sort: self.sort, selection: self.selection.clone() };
self.save_view();
let Some(lib) = self.library.as_mut() else { return Ok(()) };
lib.journal.snapshot(&self.catalog)?;
if let Ok(v) = serde_json::to_vec_pretty(&view) {
let _ = write_atomic(&lib.dir, "view.json", &v);
}
Ok(())
}
fn view_json(&self) -> Vec<u8> {
let view = ViewFile { source: self.source, filter: self.filter.clone(), sort: self.sort, selection: self.selection.clone() };
serde_json::to_vec_pretty(&view).unwrap_or_default()
}
/// Save the view state (source, filter, sort, selection) if it changed since it was last
/// written. Native hosts get this from [`Session::close_library`]; the browser host calls it
/// periodically, since a tab can be closed without notice.
pub fn save_view(&mut self) {
if self.library.is_none() {
return;
}
let v = self.view_json();
let Some(lib) = self.library.as_mut() else { return };
if v != lib.view_written {
match lib.files.write_atomic("view.json", &v) {
Ok(()) => lib.view_written = v,
Err(e) => log::error!("library: view: {e}"),
}
}
}
/// Save the library preferences (no-op for in-memory sessions).
pub fn save_prefs(&self) -> Result<()> {
let Some(lib) = &self.library else { return Ok(()) };
pub fn save_prefs(&mut self) -> Result<()> {
let v = serde_json::to_vec_pretty(&PrefsFile { xmp: self.xmp }).unwrap_or_default();
write_atomic(&lib.dir, "prefs.json", &v).map_err(|e| EngineError::Other(format!("prefs: {e}")))
let Some(lib) = self.library.as_mut() else { return Ok(()) };
lib.files.write_atomic("prefs.json", &v).map_err(|e| EngineError::Other(format!("prefs: {e}")))
}
/// Compact the log into a snapshot now.