Merge PR #842: Read large Lightroom catalogs by page

crates/engine/src/lightroom_catalog.rs: test helpers chained; native_catalog_rows takes #842's last_page argument, native_catalog_with_last_table_page builds on it
docs/parity-checklist.md: LRC-LIB-CATALOG-IMPORT keeps both texts

crates/engine/src/lightroom_catalog.rs

@@ -8,7 +8,7 @@ use std::sync::{
     atomic::{AtomicBool, AtomicUsize, Ordering},
 };
 
-use crate::lightroom_sqlite::{Database, LiveTable, Value as SqlValue};
+use crate::lightroom_sqlite::{Database, FileSource, LiveTable, Value as SqlValue};
 use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source};
 use lightcraft_geom::{Affine, Orientation, Point, Rect};
 use serde::{Deserialize, Serialize};
@@ -447,15 +447,15 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize,
     let before = std::fs::metadata(path).map_err(|e| e.to_string())?;
     let wal_path = PathBuf::from(format!("{}-wal", path.display()));
     let wal_before = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));
-    let main = bytes(path)?;
+    let main = FileSource::open(path)?;
     let wal = match std::fs::metadata(&wal_path) {
         Ok(_) => bytes(&wal_path)?,
         Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(),
         Err(e) => return Err(e.to_string()),
     };
-    let wal_len = committed_wal_len(&main, &wal)?;
+    let wal_len = committed_wal_len(main.header(), &wal)?;
     ensure_read_active(cancel)?;
-    let db = Database::open_with_wal(main, &wal[..wal_len]).map_err(|e| format!("invalid SQLite catalog: {e:?}"))?;
+    let db = Database::open_file_with_wal(main, &wal[..wal_len]).map_err(|e| format!("invalid SQLite catalog: {e:?}"))?;
     let after = std::fs::metadata(path).map_err(|e| e.to_string())?;
     let wal_after = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));
     if before.len() != after.len() || before.modified().ok() != after.modified().ok() || wal_before != wal_after {
@@ -613,6 +613,11 @@ pub fn read_with_progress(path: &Path, cancel: &AtomicBool, total: &AtomicUsize,
     drop_copies_of(&mut out, &orphans);
     refuse_non_media(&mut out);
     ensure_read_active(cancel)?;
+    reader.db.ensure_unchanged()?;
+    let wal_after = std::fs::metadata(&wal_path).ok().map(|m| (m.len(), m.modified().ok()));
+    if wal_before != wal_after {
+        return Err("catalog changed during reading; close Lightroom and retry".into());
+    }
     Ok(out)
 }
 
@@ -1592,11 +1597,16 @@ mod tests {
     }
     /// A one-row-per-table SQLite file: `(name, columns, row)` each.
     fn native_catalog(tables: &[(&str, &str, Vec<SqlValue>)]) -> Vec<u8> {
-        let tables: Vec<_> = tables.iter().map(|(name, columns, row)| (*name, *columns, vec![row.clone()])).collect();
-        native_catalog_rows(&tables)
+        native_catalog_with_last_table_page(tables, None)
     }
-    /// A SQLite file with one leaf page per table: `(name, columns, rows)` each.
-    fn native_catalog_rows(tables: &[(&str, &str, Vec<Vec<SqlValue>>)]) -> Vec<u8> {
+    /// [`native_catalog`] whose last table claims `last_page` as its root page.
+    fn native_catalog_with_last_table_page(tables: &[(&str, &str, Vec<SqlValue>)], last_page: Option<i64>) -> Vec<u8> {
+        let tables: Vec<_> = tables.iter().map(|(name, columns, row)| (*name, *columns, vec![row.clone()])).collect();
+        native_catalog_rows(&tables, last_page)
+    }
+    /// A SQLite file with one leaf page per table: `(name, columns, rows)` each. `last_page` overrides the last
+    /// table's root page.
+    fn native_catalog_rows(tables: &[(&str, &str, Vec<Vec<SqlValue>>)], last_page: Option<i64>) -> Vec<u8> {
         use SqlValue::{Integer as I, Text as T};
         let mut bytes = vec![0u8; 5 * 2048];
         bytes[..16].copy_from_slice(b"SQLite format 3\0");
@@ -1615,7 +1625,7 @@ mod tests {
                 T("table".into()),
                 T((*name).into()),
                 T((*name).into()),
-                I(i as i64 + 2),
+                I(if i + 1 == tables.len() { last_page.unwrap_or(i as i64 + 2) } else { i as i64 + 2 }),
                 T(format!("CREATE TABLE {name} ({columns})")),
             ]);
             leaf(&mut bytes[(i + 1) * 2048..(i + 2) * 2048], 0, rows);
@@ -1652,6 +1662,29 @@ mod tests {
         assert_eq!(catalog.photos[0].uuid, "fixture-image");
         assert_eq!(number(&catalog.photos[0].image, "rating"), 5);
         assert!(catalog.photos[0].path.ends_with("test.ARW"));
+        // Put the required image table beyond the old 1 GiB cutoff without allocating it.
+        // A main-file-sized allocation would defeat the purpose of the page-backed reader.
+        // The large fixture relies on Unix sparse-file behavior; small parity runs everywhere.
+        #[cfg(unix)]
+        {
+            let far_offset = 1u64 << 30;
+            let bytes = native_catalog_with_last_table_page(&photo_tables(("/catalog/", "")), Some((far_offset / 2048 + 1) as i64));
+            use std::io::{Seek, SeekFrom, Write};
+            let mut sparse = std::fs::OpenOptions::new().write(true).truncate(true).open(&path).unwrap();
+            sparse.write_all(&bytes).unwrap();
+            sparse.set_len(far_offset + 2048).unwrap();
+            sparse.seek(SeekFrom::Start(far_offset)).unwrap();
+            sparse.write_all(&bytes[4 * 2048..5 * 2048]).unwrap();
+            drop(sparse);
+            let large = read(&path).unwrap();
+            assert_eq!(serde_json::to_value(&large.photos).unwrap(), serde_json::to_value(&catalog.photos).unwrap());
+            let mut session = crate::Session::new();
+            session.execute("library.importLightroom", &json!({"path": path.to_string_lossy()})).unwrap();
+            assert_eq!(session.catalog.len(), 1);
+            assert_eq!(session.catalog.photos().next().unwrap().rating, 5);
+            session.execute("edit.undo", &json!({})).unwrap();
+            assert!(session.catalog.is_empty());
+        }
         std::fs::write(&path, b"bad catalog").unwrap();
         assert!(read(&path).is_err());
         std::fs::remove_file(path).unwrap();
@@ -1660,7 +1693,7 @@ mod tests {
     /// Reads `tables` as a catalog file and returns what the reader made of it.
     fn read_tables(tables: &[(&str, &str, Vec<Vec<SqlValue>>)], name: &str) -> CatalogImport {
         let path = std::env::temp_dir().join(format!("lightcraft-{name}-{}.lrcat", std::process::id()));
-        std::fs::write(&path, native_catalog_rows(tables)).unwrap();
+        std::fs::write(&path, native_catalog_rows(tables, None)).unwrap();
         let catalog = read(&path).unwrap();
         std::fs::remove_file(path).unwrap();
         catalog

crates/engine/src/lightroom_sqlite.rs

@@ -6,7 +6,79 @@
 //! than a SQL engine. It never writes its input and has no SQLite runtime
 //! dependency.
 
+use std::borrow::Cow;
 use std::collections::{HashMap, HashSet};
+use std::fs::{File, Metadata};
+use std::io::{Read, Seek, SeekFrom};
+use std::path::{Path, PathBuf};
+use std::sync::Mutex;
+
+const MAX_DATABASE_BYTES: u64 = 16 << 30;
+
+/// A bounded, read-only page source. Only referenced pages are loaded, never the whole file.
+pub struct FileSource {
+    file: Mutex<File>,
+    path: PathBuf,
+    before: Metadata,
+    header: Vec<u8>,
+}
+
+impl FileSource {
+    pub fn open(path: &Path) -> Result<Self, String> {
+        if !std::fs::metadata(path).map_err(|e| e.to_string())?.is_file() {
+            return Err("SQLite catalog must be a regular file".into());
+        }
+        let mut file = File::open(path).map_err(|e| e.to_string())?;
+        let before = file.metadata().map_err(|e| e.to_string())?;
+        if !before.is_file() {
+            return Err("SQLite catalog must be a regular file".into());
+        }
+        if before.len() > MAX_DATABASE_BYTES {
+            return Err("catalog exceeds 16 GiB import limit".into());
+        }
+        let mut header = vec![0; 100];
+        file.read_exact(&mut header).map_err(|e| format!("cannot read SQLite header: {e}"))?;
+        let source = Self { file: Mutex::new(file), path: path.into(), before, header };
+        source.ensure_unchanged()?;
+        Ok(source)
+    }
+
+    pub fn header(&self) -> &[u8] {
+        &self.header
+    }
+
+    pub fn ensure_unchanged(&self) -> Result<(), String> {
+        let file = self.file.lock().map_err(|_| "SQLite page source lock poisoned")?;
+        let handle = file.metadata().map_err(|e| e.to_string())?;
+        let path = std::fs::metadata(&self.path).map_err(|_| "catalog changed during reading; close Lightroom and retry")?;
+        if !same_metadata(&self.before, &handle) || !same_metadata(&self.before, &path) {
+            return Err("catalog changed during reading; close Lightroom and retry".into());
+        }
+        Ok(())
+    }
+
+    fn page(&self, page_id: u32, size: usize) -> Result<Vec<u8>, String> {
+        let start = u64::from(page_id).checked_sub(1).and_then(|n| n.checked_mul(size as u64)).ok_or("page offset overflow")?;
+        if start.checked_add(size as u64).is_none_or(|end| end > self.before.len()) {
+            return Err(format!("page {page_id} is truncated"));
+        }
+        let mut out = vec![0; size];
+        let mut file = self.file.lock().map_err(|_| "SQLite page source lock poisoned")?;
+        file.seek(SeekFrom::Start(start)).and_then(|_| file.read_exact(&mut out)).map_err(|e| format!("cannot read SQLite page {page_id}: {e}"))?;
+        Ok(out)
+    }
+}
+
+fn same_metadata(a: &Metadata, b: &Metadata) -> bool {
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::MetadataExt;
+        if a.dev() != b.dev() || a.ino() != b.ino() {
+            return false;
+        }
+    }
+    a.len() == b.len() && a.modified().ok() == b.modified().ok() && a.created().ok() == b.created().ok()
+}
 
 const MAX_PAGE_ID: u32 = 16_777_216;
 const MAX_COLUMNS: usize = 16_384;
@@ -44,6 +116,7 @@ pub struct LiveTable {
 /// Read-only SQLite database image, with a committed WAL overlay if supplied.
 pub struct Database {
     main: Vec<u8>,
+    file: Option<FileSource>,
     overlay: HashMap<u32, Vec<u8>>,
     page_size: usize,
     reserved: usize,
@@ -84,17 +157,34 @@ impl Database {
     /// The WAL argument must include its 32-byte header and only complete
     /// frames through a commit. Frames are overlaid by page number, with the
     /// final commit's database-size field defining the visible page count.
+    #[cfg(test)]
     pub fn open_with_wal(main: Vec<u8>, committed_wal: &[u8]) -> Result<Self, String> {
+        Self::open(main, None, committed_wal)
+    }
+
+    pub fn open_file_with_wal(source: FileSource, committed_wal: &[u8]) -> Result<Self, String> {
+        Self::open(source.header().to_vec(), Some(source), committed_wal)
+    }
+
+    pub fn ensure_unchanged(&self) -> Result<(), String> {
+        if let Some(file) = &self.file {
+            file.ensure_unchanged()?;
+        }
+        Ok(())
+    }
+
+    fn open(main: Vec<u8>, file: Option<FileSource>, committed_wal: &[u8]) -> Result<Self, String> {
         let main_header = main.get(..100).ok_or("truncated SQLite header")?;
         if main_header.get(..16) != Some(b"SQLite format 3\0") {
             return Err("invalid SQLite header signature".into());
         }
         let page_size = sqlite_page_size(be_u16(main_header, 16)?)?;
         validate_header_fields(main_header, page_size)?;
-        if main.len() < page_size || !main.len().is_multiple_of(page_size) {
+        let main_len = file.as_ref().map_or(main.len() as u64, |source| source.before.len());
+        if main_len < page_size as u64 || !main_len.is_multiple_of(page_size as u64) {
             return Err("SQLite image is not a whole number of pages".into());
         }
-        let main_pages = u32::try_from(main.len() / page_size).map_err(|_| "SQLite image is too large")?;
+        let main_pages = u32::try_from(main_len / page_size as u64).map_err(|_| "SQLite image is too large")?;
         if main_pages == 0 || main_pages > MAX_PAGE_ID {
             return Err("invalid SQLite image page count".into());
         }
@@ -140,7 +230,7 @@ impl Database {
             _ => return Err("unsupported SQLite text encoding".into()),
         };
 
-        let mut db = Self { main, overlay, page_size, reserved, encoding, page_count, tables: Vec::new(), rowid_aliases: HashMap::new() };
+        let mut db = Self { main, file, overlay, page_size, reserved, encoding, page_count, tables: Vec::new(), rowid_aliases: HashMap::new() };
         let schema = db.parse_schema()?;
         db.tables = schema.tables;
         db.rowid_aliases = schema.aliases;
@@ -166,17 +256,20 @@ impl Database {
         Ok(state.out)
     }
 
-    fn page(&self, page_id: u32) -> Result<&[u8], String> {
+    fn page(&self, page_id: u32) -> Result<Cow<'_, [u8]>, String> {
         if page_id == 0 || page_id > self.page_count {
             return Err(format!("page {page_id} is outside database"));
         }
         if let Some(page) = self.overlay.get(&page_id) {
-            return Ok(page);
+            return Ok(Cow::Borrowed(page));
+        }
+        if let Some(file) = &self.file {
+            return file.page(page_id, self.page_size).map(Cow::Owned);
         }
         let id = usize::try_from(page_id).map_err(|_| "page number does not fit host")?;
         let start = id.checked_sub(1).and_then(|n| n.checked_mul(self.page_size)).ok_or("page offset overflow")?;
         let end = start.checked_add(self.page_size).ok_or("page offset overflow")?;
-        self.main.get(start..end).ok_or_else(|| format!("page {page_id} is truncated"))
+        self.main.get(start..end).map(Cow::Borrowed).ok_or_else(|| format!("page {page_id} is truncated"))
     }
 
     fn visit_table(&self, page_id: u32, ncols: usize, alias: Option<usize>, depth: usize, state: &mut WalkState) -> Result<(), String> {
@@ -219,7 +312,7 @@ impl Database {
                     if state.payload_total > MAX_TABLE_PAYLOAD {
                         return Err("SQLite table payload exceeds bounded import size".into());
                     }
-                    let payload = self.payload(page, at, payload_len)?;
+                    let payload = self.payload(&page, at, payload_len)?;
                     let mut values = decode_record(&payload, ncols, self.encoding)?;
                     if let Some(index) = alias {
                         if index >= values.len() {
@@ -325,7 +418,7 @@ impl Database {
             out.extend_from_slice(overflow.get(4..end).ok_or("truncated SQLite overflow page")?);
             remaining -= take;
             if remaining > 0 {
-                next = be_u32_at(overflow, 0)?;
+                next = be_u32_at(&overflow, 0)?;
             }
         }
         Ok(out)
@@ -873,6 +966,72 @@ fn be_u32_at(bytes: &[u8], at: usize) -> Result<u32, String> {
 mod tests {
     use super::*;
 
+    struct Scratch(PathBuf);
+    impl Scratch {
+        fn new(bytes: &[u8]) -> Self {
+            static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+            let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+            let path = std::env::temp_dir().join(format!("lightcraft-page-source-{}-{n}.lrcat", std::process::id()));
+            std::fs::write(&path, bytes).unwrap();
+            Self(path)
+        }
+    }
+    impl Drop for Scratch {
+        fn drop(&mut self) {
+            let _ = std::fs::remove_file(&self.0);
+        }
+    }
+
+    fn file_backed(mut db: Database) -> (Scratch, Database) {
+        let scratch = Scratch::new(&db.main);
+        db.file = Some(FileSource::open(&scratch.0).unwrap());
+        db.main.truncate(100);
+        (scratch, db)
+    }
+
+    #[test]
+    fn file_source_bounds_and_mutation_detection() {
+        let scratch = Scratch::new(&[0; 512]);
+        let source = FileSource::open(&scratch.0).unwrap();
+        assert_eq!(source.page(1, 512).unwrap().len(), 512);
+        assert!(source.page(2, 512).unwrap_err().contains("truncated"));
+        File::options().write(true).open(&scratch.0).unwrap().set_len(100).unwrap();
+        assert!(source.page(1, 512).is_err());
+        assert!(source.ensure_unchanged().unwrap_err().contains("changed during reading"));
+        drop(source);
+        // Unix set_len creates a hole. Do not reserve a 16 GiB normal file on Windows.
+        #[cfg(unix)]
+        {
+            File::options().write(true).open(&scratch.0).unwrap().set_len(MAX_DATABASE_BYTES + 1).unwrap();
+            assert_eq!(FileSource::open(&scratch.0).err().unwrap(), "catalog exceeds 16 GiB import limit");
+        }
+    }
+
+    #[cfg(unix)]
+    #[test]
+    fn file_source_detects_same_sized_path_replacement() {
+        let original = Scratch::new(&[0; 512]);
+        let replacement = Scratch::new(&[0; 512]);
+        let source = FileSource::open(&original.0).unwrap();
+        std::fs::rename(&replacement.0, &original.0).unwrap();
+        assert!(source.ensure_unchanged().unwrap_err().contains("changed during reading"));
+    }
+
+    #[test]
+    fn file_source_detects_same_length_content_changes() {
+        let scratch = Scratch::new(&[0; 512]);
+        let source = FileSource::open(&scratch.0).unwrap();
+        std::fs::write(&scratch.0, [1; 512]).unwrap();
+        // Make the timestamp change deterministic even on filesystems with coarse timestamps.
+        File::options()
+            .write(true)
+            .open(&scratch.0)
+            .unwrap()
+            .set_modified(source.before.modified().unwrap() + std::time::Duration::from_secs(2))
+            .unwrap();
+        assert!(source.ensure_unchanged().unwrap_err().contains("changed during reading"));
+    }
+
     fn varint(mut value: u64) -> Vec<u8> {
         let mut out = vec![u8::try_from(value & 0x7f).unwrap_or(0)];
         value >>= 7;
@@ -1026,6 +1185,40 @@ mod tests {
         assert!(tables.iter().any(|table| table.name == "t"));
         let rows = db.read_table(2, 2).unwrap();
         assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(1)));
+        let scratch = Scratch::new(&db.main);
+        let file_db = Database::open_file_with_wal(FileSource::open(&scratch.0).unwrap(), &[]).unwrap();
+        assert_eq!(file_db.live_tables().unwrap(), tables);
+        assert_eq!(file_db.read_table(2, 2).unwrap(), rows);
+        assert_eq!(file_db.main.len(), 100);
+        file_db.ensure_unchanged().unwrap();
+        // The same committed WAL page overrides the lazily read main page.
+        let mut wal = vec![0; 32 + 24 + 512];
+        wal[..4].copy_from_slice(&0x377f0682u32.to_be_bytes());
+        wal[4..8].copy_from_slice(&3_007_000u32.to_be_bytes());
+        wal[8..12].copy_from_slice(&512u32.to_be_bytes());
+        wal[32..36].copy_from_slice(&2u32.to_be_bytes());
+        wal[36..40].copy_from_slice(&2u32.to_be_bytes());
+        leaf(&mut wal[56..], 0, 2, &[3, 0, 15, b'y']);
+        let memory_wal = Database::open_with_wal(db.main.clone(), &wal).unwrap();
+        let file_wal = Database::open_file_with_wal(FileSource::open(&scratch.0).unwrap(), &wal).unwrap();
+        assert_eq!(memory_wal.read_table(2, 2).unwrap(), file_wal.read_table(2, 2).unwrap());
+        assert_eq!(file_wal.read_table(2, 2).unwrap()[0].values[0], Value::Integer(2));
+        let mut malformed = db.main.clone();
+        malformed.push(0);
+        let partial = Scratch::new(&malformed);
+        assert_eq!(
+            Database::open_file_with_wal(FileSource::open(&partial.0).unwrap(), &[]).err().unwrap(),
+            "SQLite image is not a whole number of pages"
+        );
+        let mut malformed = db.main.clone();
+        malformed[24..28].copy_from_slice(&1u32.to_be_bytes());
+        malformed[92..96].copy_from_slice(&1u32.to_be_bytes());
+        malformed[28..32].copy_from_slice(&3u32.to_be_bytes());
+        let oversized = Scratch::new(&malformed);
+        assert_eq!(
+            Database::open_file_with_wal(FileSource::open(&oversized.0).unwrap(), &[]).err().unwrap(),
+            "SQLite header page count exceeds image"
+        );
     }
 
     #[test]
@@ -1042,6 +1235,7 @@ mod tests {
         leaf(&mut main[1024..1536], 0, 1, &[2, 0]);
         leaf(&mut main[1536..], 0, 2, &[2, 0]);
         let db = Database {
+            file: None,
             main,
             overlay: HashMap::new(),
             page_size: 512,
@@ -1073,6 +1267,7 @@ mod tests {
         let mut aliases = HashMap::new();
         aliases.insert(1, 0);
         let db = Database {
+            file: None,
             main,
             overlay: HashMap::new(),
             page_size: 512,
@@ -1093,6 +1288,7 @@ mod tests {
         main[103..105].copy_from_slice(&1u16.to_be_bytes());
         main[108..110].copy_from_slice(&490u16.to_be_bytes());
         let db = Database {
+            file: None,
             main,
             overlay: HashMap::new(),
             page_size: 512,
@@ -1103,6 +1299,8 @@ mod tests {
             rowid_aliases: HashMap::new(),
         };
         assert!(db.read_table(1, 1).is_err());
+        let (_scratch, db) = file_backed(db);
+        assert!(db.read_table(1, 1).is_err());
     }
 
     #[test]
@@ -1115,6 +1313,7 @@ mod tests {
 
         let mut main = vec![0u8; 2 * 512];
         let db = Database {
+            file: None,
             main: std::mem::take(&mut main),
             overlay: HashMap::new(),
             page_size: 512,
@@ -1141,6 +1340,10 @@ mod tests {
         assert_eq!(bytes.len(), 600);
         assert_eq!(&bytes[..92], &[b'x'; 92]);
         assert_eq!(&bytes[92..], &[b'y'; 508]);
+        let (_scratch, mut db) = file_backed(db);
+        db.overlay.clear();
+        let bytes = db.payload(&cell_page, 0, 600).unwrap();
+        assert_eq!(&bytes[92..], &[b'y'; 508]);
     }
 
     #[test]
@@ -1164,6 +1367,7 @@ mod tests {
         let mut page = vec![0u8; 512];
         page[16..18].copy_from_slice(&512u16.to_be_bytes());
         let db = Database {
+            file: None,
             main: vec![0; 512],
             overlay: HashMap::new(),
             page_size: 512,

docs/gaps.md

@@ -27,7 +27,7 @@ Hours are Opus 5.5 agent wall-clock hours (see [target-app-parity.md](target-app
 | 10 B | **Per-model verification thin, plus cheap per-body decode fixes**: 545 of 1,446 Adobe models verified (38%), ~85% of photographers on a verified body; Nikon 5 bodies verified, Canon 34, Leica 14 (the Q3 by a recorded check of 38 private DNGs). Cheap fixes: Z 6II packed 14-bit NEF (#769, PR #671), Sony A100 / F828 regression (#702), DSLR-A200…A390 / A850 12-bit ARW (#535) | measured: [raw/coverage.tsv](raw/coverage.tsv) | per-body bugs (levels, crops, WB) surface only from users | 25–45 | [raw-parity](raw-parity.md#by-maker-measured) |
 | 11 | **Enhance: Raw Details, Super Resolution; AI denoise without shippable weights or X-Trans** | ENH rows 0%; denoise Bayer only, RawNIND weights GPL and opt-in | Fujifilm users get no AI denoise; nobody gets it out of the box | 30–60 + model decision | [raw-parity](raw-parity.md#demosaic-levels-crops-and-lens-metadata) |
 | 12 | **Content-aware / generative remove, detect distractions, reflections removal** | REM rows 75%; patch synthesis missing | cleanup work goes back to Photoshop | 25–45 (+ generative model) | [target-app-parity](target-app-parity.md#ai) |
-| 13 | **Lightroom catalog migration is one-way and approximate**: unsupported Adobe profiles / AI settings archived, history kept as data; offline record counts and destination are now shown in a lasting import result | lightroom-catalog-import.md | switching users lose some looks | 8–12 | [file-format-parity](file-format-parity.md) |
+| 13 | **Lightroom catalog migration is one-way and approximate**: unsupported Adobe profiles / AI settings archived, history kept as data; main-file pages read on demand up to 16 GiB, with dense-table and WAL bounds retained (#798); offline record counts and destination are now shown in a lasting import result | lightroom-catalog-import.md | switching users lose some looks | 8–12 | [file-format-parity](file-format-parity.md) |
 | 14 | **HDR display** (EDR / Windows HDR); the headroom limit and SDR rendition from `crs` XMP (the HDR switch, `HDREditMode`, is read). HLG AVIF export done; the display's SDR/HDR decision is tested, blocked on eframe's 8-bit surface | HDR rows 70%; LR-VIEW-HDR-DISPLAY ⬜ | HDR editing works blind on HDR monitors | 12–25 | [hardware-parity](hardware-parity.md) |
 | 15 | **Print module**: identity plate / watermark / photo info overlays, Cut Guides for the packages, Custom Package Auto Layout, printer-driver Page Setup and direct printing, JPEG file output (the Print dialog writes a colour-managed PDF) | LRC-PRINT rows | Classic users who print | 15–25 | [target-app-parity](target-app-parity.md#features) |
 | 16 | **Map view, Book, Slideshow module, publish services** | LRC-MAP / BOOK / SS rows ⬜ | Classic users; publish services matter for Flickr / SmugMug workflows | 60–100 | [target-app-parity](target-app-parity.md#features) |

docs/lightroom-catalog-import.md

@@ -49,11 +49,16 @@ retained. Oldest managed archives are retired; legacy `.json` archives remain un
 `Interop/lightroom-index.json` records imported identities. Source catalogs and originals are
 never overwritten.
 
-Each native `.lrcat` database and WAL file is independently limited to 1 GiB
-(1,073,741,824 bytes). This bounds the current whole-file reader; referenced photo sizes and
-photo counts are not part of that limit. XMP packets are capped at 16 MiB and each table at one million rows.
-The in-tree, read-only Rust SQLite B-tree reader follows SQLite’s public file-format
-specification; it introduces no dependency or C runtime. Table payloads are capped at 256 MiB.
+Native import reads referenced database pages on demand (at most 64 KiB per page), rather than
+loading the whole `.lrcat`. Database files are capped at 16 GiB and 16,777,216 pages; the WAL
+remains capped at 1 GiB. Tables still have a 256 MiB encoded-payload budget and one million rows,
+individual records are capped at 64 MiB, and XMP packets at 16 MiB. These limits can still reject
+large catalogs with dense tables; file size alone no longer determines the memory allocation.
+The open file and source path are checked again after all tables are read. Close Lightroom before
+importing: metadata checks detect ordinary concurrent changes, but are not SQLite snapshot isolation.
+
+The in-tree, read-only Rust SQLite B-tree reader follows SQLite's public file-format
+specification; it introduces no dependency or C runtime.
 Optional history, keyword and collection tables that exceed limits are skipped with warnings;
 required photo tables still report errors. Unrelated unsupported schemas are ignored. A needed
 WITHOUT ROWID or virtual table produces an explicit error, handled as a warning when optional.
@@ -111,3 +116,7 @@ an absolute path is not looked at: the import lists it as missing with a warning
 photo and its virtual copies out of the library. To import originals on a share, open the catalog
 from that share. A path that is not a path on this computer (a Windows drive path read on macOS or
 Linux) is not looked at either and stays catalogued as missing.
+
+## Historical local validation
+
+Validated in a local Linux environment after upstream reconciliation: 47 catalog-focused tests, nine pipeline spot tests, the exact sparse-catalog regression, and all eight `cargo xtask ci` stages passed. The full gate recorded 3,166 passing test executions, zero failures, and 28 ignored tests. An original synthetic sparse catalog with a required image-table page beyond 1 GiB also passed headless application import and selection; its expected missing-original placeholder retained its five-star rating. This snapshot check does not establish native-window coverage or dense-catalog performance.

docs/parity-checklist.md

@@ -698,7 +698,7 @@ From `08-lightroom-classic-extras.md` (Classic-only features) and part 2 of `06-
 | LRC-LIB-SURVEY | Survey view | P2 | ✅ | `cmd:view.survey` | N |
 | LRC-LIB-REFVIEW | Reference view | P2 | ✅ | `cmd:view.reference`, `cmd:photo.setReference`, `crates/ui-egui/src/panels/compare.rs` (`show_reference`) | View ▸ Reference View (⇧R): the reference photo beside the active one (edited with the Edit panel); photo menu ▸ Set as Reference Photo; each pane says “Rendering…” (widget `compare:rendering:<pane>`) or “This photo can't be opened” (`compare:renderFailed:<pane>`) as the loupe does |
 | LRC-LIB-CATALOG-PANEL | Catalog sets | P1 | ✅ | `cmd:library.source` (`missing`) | all, recently added, picks, recently deleted, missing photos (when any) |
-| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages (also WAL-only tables with main schema format 0), in-tree read-only Rust SQLite B-tree reader; background inspection/import with progress/cancellation, per-open library identity checks; persistent result with destination library, offline-original counts and Show in Library, dismissed when switching libraries; offline Local records are promoted without probing missing files; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, a root folder not found at its recorded path looked up relative to the catalog (Lightroom's `relativePathFromCatalog`, so a project folder copied from another computer stays linked; reported as a warning), existing edits preserved. Best-effort deduplicated recovery archives retain settings/history/snapshots/collection rules (32 MiB each, eight archives/128 MiB aggregate); archive limits/write failures and optional-table limits warn without blocking photo import; unrelated unsupported tables and opaque SQL BLOBs skipped. Custom white balance on relative-WB raws (`RAW` in Lightroom's terms) stays As Shot and is reported, since the catalog keeps no as-shot white to shift from (issue #510). An image whose file, folder or root folder record is missing from the catalog, or whose original path is empty, is skipped with its virtual copies instead of failing the whole import; the report lists them as data (`skipped`: image id and reason) and the result window counts them in the UI language (PR #2230). Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs. Open Library given a Lightroom catalog folder, an `.lrdata` / `.lrcat-data` folder or the `.lrcat` makes no library there and offers this import with the catalog found (`cmd:app.openLibrary`, `crates/engine/src/lightroom_folder.rs`; issue #891) |
+| LRC-LIB-CATALOG-IMPORT | Native Lightroom Classic catalog import | P1 | 🟡 | `cmd:library.inspectLightroom`, `cmd:library.importLightroom`, `crates/engine/src/lightroom_catalog.rs`, `docs/lightroom-catalog-import.md` | read-only `.lrcat` plus committed WAL pages (also WAL-only tables with main schema format 0), in-tree read-only Rust SQLite B-tree reader; on-demand main-file pages up to 16 GiB with existing table/record bounds (WAL still capped at 1 GiB, issue #798); background inspection/import with progress/cancellation, per-open library identity checks; persistent result with destination library, offline-original counts and Show in Library, dismissed when switching libraries; offline Local records are promoted without probing missing files; originals in place, ratings/flags/labels, XMP, hierarchical keywords, collection sets/regular collections, virtual copies, mapped develop settings; stable reimport identities, undo, missing-file placeholders, a root folder not found at its recorded path looked up relative to the catalog (Lightroom's `relativePathFromCatalog`, so a project folder copied from another computer stays linked; reported as a warning), existing edits preserved. Best-effort deduplicated recovery archives retain settings/history/snapshots/collection rules (32 MiB each, eight archives/128 MiB aggregate); archive limits/write failures and optional-table limits warn without blocking photo import; unrelated unsupported tables and opaque SQL BLOBs skipped. Custom white balance on relative-WB raws (`RAW` in Lightroom's terms) stays As Shot and is reported, since the catalog keeps no as-shot white to shift from (issue #510). An image whose file, folder or root folder record is missing from the catalog, or whose original path is empty, is skipped with its virtual copies instead of failing the whole import; the report lists them as data (`skipped`: image id and reason) and the result window counts them in the UI language (PR #2230). Adobe rendering, AI data and native history replay remain incomplete; no writes to Lightroom catalogs. Open Library given a Lightroom catalog folder, an `.lrdata` / `.lrcat-data` folder or the `.lrcat` makes no library there and offers this import with the catalog found (`cmd:app.openLibrary`, `crates/engine/src/lightroom_folder.rs`; issue #891) |
 | LRC-LIB-CATALOG-LOCK | Catalog open in one program at a time | P1 | ✅ | `cmd:app.openLibrary`, `crates/catalog/src/lock.rs`, `crates/engine/src/library.rs` (`open_library`) | exclusive OS lock on `catalog.lock` for the session (app and `lightcraft-cli --library`); a second opener is refused with who has it (`catalog.lock.owner`: program, process, computer); released by the OS on a crash, so never stale; file systems without locks open unprotected with a warning (the app's notices, `library.info` → `settingsWarnings`, and the CLI's stderr; issue #171); no read-only mode; the browser build (OPFS) has no guard across tabs yet. A library that can't be opened at launch (locked, unreadable, newer format, drive missing) shows a window — Try Again, Choose Another Library…, Continue Without Saving (then a banner; nothing written), Quit — instead of a silent demo session (`crates/ui-egui/src/panels/library_problem.rs`); the lock is released explicitly when the library closes, not by letting its file handle close, so a program another thread is starting at that moment (which holds a copy of every file handle until it is launched) cannot keep the library "already open" for a moment (issue #680) |
 | LRC-LIB-CATALOG-SWITCH | New / Open Recent catalog, catalog name shown | P1 | ✅ | `cmd:app.newLibrary`, `cmd:app.openLibrary`, `cmd:app.recentLibraries`, `cmd:app.clearRecentLibraries`, `crates/ui-egui/src/panels/settings.rs`, `crates/ui-egui/src/panels/left.rs` | File ▸ New Library… (name, then location), File ▸ Open Recent Library (last 8; missing ones greyed out, dropped when opened; Clear), the open library's name under My Photos and in the window title (issue #902); a library that can't be opened leaves the current one open; ⌘O (Ctrl+O on Windows and Linux) runs Open Library… and asks for a folder once (`crates/ui-egui/src/tests_keymap.rs`: `cmd_o_opens_the_library_picker_once`); a dialog or a text field with the focus keeps the key |
 | LRC-LIB-FOLDERS | Disk folder tree | P1 | ✅ | `cmd:library.browse`, `cmd:folder.rename`, `cmd:folder.move`, `crates/ui-egui/src/panels/left.rs` (`local_section`, `folder_tree`) | standard places + the browsed folder + Browse Folder…, each an expandable tree of subfolders, breadcrumb navigation; context menu: Rename Folder…, Move Folder To…, Show in Finder (on disk, photos relinked; one undo step each: undo / redo move the folder back and forth, refused when the destination is taken) |