Merge PR #1662

2026-11-02 14:39:41 -06:00 · 2026-10-14 22:32:55 -04:00
parent 31f1f559dd 43ba500c6e
commit 9562532695
5 changed files with 354 additions and 20 deletions
@@ -25,6 +25,7 @@ pub mod rules;
 pub mod safe_file;
 pub mod stacks;
 pub mod store;
+pub mod tally;

 use std::collections::BTreeMap;
 use std::sync::Arc;
@@ -0,0 +1,234 @@
+//! How many photos each album shows, kept up to date from what the latest ops touched.
+//!
+//! A smart album's count asks every photo whether its rules hold. Asked again after every edit, that is a pass over
+//! the library for each smart album each time a keyword is given. [`AlbumTally`] remembers which photos each smart
+//! album holds, and after a small edit asks only the photos the edit touched ([`Catalog::touched_since`]). An op
+//! that changes more than photos of its own (an album's rules or photos, a stack, the keyword list) has it counted
+//! whole again, as before.
+
+use std::collections::{HashMap, HashSet};
+
+use crate::{AlbumId, Aspects, Catalog, PhotoId};
+
+/// The most photos an edit may touch for the counts to follow it photo by photo.
+const MAX_FOLLOWED: usize = 256;
+/// Bits are kept while they cost no more than this many words a photo held, and these to start with. Past that
+/// (photo ids far apart: a library whose ids don't count up from 1) a set is kept instead, so what an album
+/// remembers is always in proportion to the photos it holds, never to the largest id in the library.
+const WORDS_A_MEMBER: usize = 4;
+const WORDS_TO_START: usize = 1024;
+
+/// The photos a smart album holds.
+#[derive(Clone, Debug)]
+enum Members {
+    /// One bit a photo id, and how many are set.
+    Bits {
+        words: Vec<u64>,
+        len: usize,
+    },
+    Set(HashSet<PhotoId>),
+}
+
+impl Members {
+    fn none() -> Members {
+        Members::Bits { words: Vec::new(), len: 0 }
+    }
+
+    fn len(&self) -> usize {
+        match self {
+            Members::Bits { len, .. } => *len,
+            Members::Set(set) => set.len(),
+        }
+    }
+
+    #[cfg(test)]
+    fn holds(&self, id: PhotoId) -> bool {
+        match self {
+            Members::Bits { words, .. } => {
+                usize::try_from(id.0 / 64).ok().and_then(|at| words.get(at)).is_some_and(|word| word & (1u64 << (id.0 % 64)) != 0)
+            }
+            Members::Set(set) => set.contains(&id),
+        }
+    }
+
+    /// Put `id` in or take it out.
+    fn set(&mut self, id: PhotoId, held: bool) {
+        if let Members::Bits { words, len } = self {
+            let bit = 1u64 << (id.0 % 64);
+            match usize::try_from(id.0 / 64) {
+                Ok(at) if at < words.len() => {
+                    if let Some(word) = words.get_mut(at) {
+                        match (*word & bit != 0, held) {
+                            (false, true) => {
+                                *word |= bit;
+                                *len = len.saturating_add(1);
+                            }
+                            (true, false) => {
+                                *word &= !bit;
+                                *len = len.saturating_sub(1);
+                            }
+                            _ => {}
+                        }
+                    }
+                    return;
+                }
+                // (past the end: not held, so there is nothing to take out)
+                _ if !held => return,
+                // room for it, as long as the bits stay in proportion to the photos held
+                Ok(at) if at < len.saturating_mul(WORDS_A_MEMBER).saturating_add(WORDS_TO_START) => {
+                    words.resize(at + 1, 0);
+                    if let Some(word) = words.get_mut(at) {
+                        *word |= bit;
+                        *len = len.saturating_add(1);
+                    }
+                    return;
+                }
+                // ids too far apart for bits: a set from here on
+                _ => {
+                    let ids = words
+                        .iter()
+                        .enumerate()
+                        .flat_map(|(at, word)| (0..64u64).filter(move |b| word & (1u64 << b) != 0).map(move |b| PhotoId(at as u64 * 64 + b)));
+                    *self = Members::Set(ids.collect());
+                }
+            }
+        }
+        if let Members::Set(set) = self {
+            if held {
+                set.insert(id);
+            } else {
+                set.remove(&id);
+            }
+        }
+    }
+}
+
+/// [`Catalog::album_count`] of every album, as of one revision of the catalog.
+#[derive(Clone, Debug)]
+pub struct AlbumTally {
+    revision: u64,
+    counts: HashMap<AlbumId, usize>,
+    /// The smart albums' photos.
+    members: HashMap<AlbumId, Members>,
+}
+
+impl AlbumTally {
+    /// Counted from every photo.
+    pub fn of(cat: &Catalog) -> AlbumTally {
+        let mut counts = HashMap::new();
+        let mut members = HashMap::new();
+        for album in cat.albums() {
+            if album.is_smart() {
+                let mut held = Members::none();
+                for p in cat.photos().filter(|p| cat.album_contains(album.id, p)) {
+                    held.set(p.id, true);
+                }
+                counts.insert(album.id, held.len());
+                members.insert(album.id, held);
+            } else {
+                counts.insert(album.id, cat.album_count(album.id));
+            }
+        }
+        AlbumTally { revision: cat.revision, counts, members }
+    }
+
+    /// The count of each album.
+    pub fn counts(&self) -> &HashMap<AlbumId, usize> {
+        &self.counts
+    }
+
+    /// The catalog's revision the counts are of.
+    pub fn revision(&self) -> u64 {
+        self.revision
+    }
+
+    /// Bring the counts up to `cat` as it is now. `true` when that was done from the photos the ops since touched
+    /// (or nothing had changed), `false` when every photo was asked again.
+    ///
+    /// `cat` has to be the catalog the counts were made of, further along. Another catalog is told apart only by
+    /// its revisions: a library that is loaded takes a range of its own (and then everything is counted again), but
+    /// two catalogs both counting from 0 look like one.
+    pub fn update(&mut self, cat: &Catalog) -> bool {
+        if self.revision == cat.revision {
+            return true;
+        }
+        let followed = match (cat.touched_since(self.revision), cat.aspects_since(self.revision)) {
+            (Some(touched), Some(changed)) if touched.len() <= MAX_FOLLOWED => {
+                self.follow(cat, &touched, changed);
+                true
+            }
+            _ => false,
+        };
+        if !followed {
+            *self = AlbumTally::of(cat);
+        }
+        followed
+    }
+
+    /// The counts after ops that touched only `touched`: each smart album asks those photos again; an album of
+    /// chosen photos counts again only when a photo came, went or was deleted (who is in it did not change: that
+    /// is an op on the album, and not followed).
+    fn follow(&mut self, cat: &Catalog, touched: &[PhotoId], changed: Aspects) {
+        for (album, held) in &mut self.members {
+            for id in touched {
+                held.set(*id, cat.photo(*id).is_some_and(|p| cat.album_contains(*album, p)));
+            }
+            self.counts.insert(*album, held.len());
+        }
+        if changed.any_of(Aspects::PRESENCE) {
+            for album in cat.albums().filter(|a| !a.is_smart()) {
+                self.counts.insert(album.id, cat.album_count(album.id));
+            }
+        }
+        self.revision = cat.revision;
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn ids(m: &Members, of: &[u64]) -> Vec<u64> {
+        of.iter().copied().filter(|id| m.holds(PhotoId(*id))).collect()
+    }
+
+    #[test]
+    fn members_are_kept_as_bits_at_the_edges_of_a_word_and_past_the_end() {
+        let mut m = Members::none();
+        let probe = [0, 1, 63, 64, 65, 127, 128, 5000];
+        assert!(ids(&m, &probe).is_empty() && m.len() == 0);
+        for id in [0, 63, 64, 5000] {
+            m.set(PhotoId(id), true);
+        }
+        assert_eq!((ids(&m, &probe), m.len()), (vec![0, 63, 64, 5000], 4));
+        // put in twice, taken out twice, taken out where nothing is, far past the end: counted once each way
+        m.set(PhotoId(63), true);
+        m.set(PhotoId(64), false);
+        m.set(PhotoId(64), false);
+        m.set(PhotoId(1), false);
+        m.set(PhotoId(900_000), false);
+        assert_eq!((ids(&m, &probe), m.len()), (vec![0, 63, 5000], 3));
+        assert!(matches!(m, Members::Bits { .. }));
+    }
+
+    #[test]
+    fn photo_ids_far_apart_are_kept_as_a_set_in_proportion_to_the_photos_held() {
+        // one photo with an id in the thousands of millions: no bit for every id up to it
+        let mut m = Members::none();
+        for id in [3, 70, 9_000_000_000] {
+            m.set(PhotoId(id), true);
+        }
+        assert!(matches!(m, Members::Set(_)), "a set once the ids are too far apart for bits");
+        assert_eq!((ids(&m, &[3, 70, 71, 9_000_000_000, u64::MAX]), m.len()), (vec![3, 70, 9_000_000_000], 3));
+        m.set(PhotoId(u64::MAX), true);
+        m.set(PhotoId(3), false);
+        assert_eq!((ids(&m, &[3, 70, 9_000_000_000, u64::MAX]), m.len()), (vec![70, 9_000_000_000, u64::MAX], 3));
+        // an album of many photos with ids counting up stays bits, however many
+        let mut dense = Members::none();
+        for id in 1..=200_000u64 {
+            dense.set(PhotoId(id), true);
+        }
+        assert!(matches!(&dense, Members::Bits { words, .. } if words.len() <= 200_000 / 64 + 1));
+        assert_eq!(dense.len(), 200_000);
+    }
+}
@@ -219,3 +219,77 @@ fn an_edit_to_many_photos_or_to_more_than_photos_is_worked_out_whole() {
    let _ = s.visible_cloned();
    assert_eq!((s.view_followed, s.view_worked_out), (followed + 1, whole + 2));
 }
+
+#[test]
+fn the_albums_counts_followed_through_edits_are_the_counts_made_from_every_photo() {
+    use lightcraft_catalog::tally::AlbumTally;
+    let (mut s, mut ids, album, smart) = library(240);
+    // more smart albums: picks, well rated or keyworded ones, one that looks into an album of chosen photos
+    for rules in [
+        json!({"ruleSet": {"match": "all", "rules": [{"field": "flag", "op": "is", "value": "pick"}]}}),
+        json!({"ruleSet": {"match": "any", "rules": [{"field": "rating", "op": "gte", "value": 4}, {"field": "keywords", "op": "contains", "value": "pets"}]}}),
+        json!({"album": album, "rating": 3}),
+        // what the edits below change besides: a label, a slider, when a photo was taken, what its file is called
+        json!({"label": "red"}),
+        json!({"edited": true}),
+        json!({"dateFrom": "2025-06-01"}),
+        json!({"text": "REN"}),
+    ] {
+        let made = s.execute("album.createSmart", &json!({"name": "More", "rules": rules}));
+        assert!(made.is_ok(), "{rules}: {made:?}");
+    }
+    assert!(s.catalog.album(lightcraft_catalog::AlbumId(smart)).is_some_and(|a| a.is_smart()));
+    let fresh = |s: &Session| -> Vec<(u64, usize)> {
+        let mut v: Vec<(u64, usize)> = s.catalog.albums().map(|a| (a.id.0, s.catalog.album_count(a.id))).collect();
+        v.sort_unstable();
+        v
+    };
+    let shown = |tally: &AlbumTally| -> Vec<(u64, usize)> {
+        let mut v: Vec<(u64, usize)> = tally.counts().iter().map(|(id, n)| (id.0, *n)).collect();
+        v.sort_unstable();
+        v
+    };
+    let mut tally = AlbumTally::of(&s.catalog);
+    assert_eq!(shown(&tally), fresh(&s));
+    let mut dice = Dice(23);
+    let (mut followed, mut whole) = (0, 0);
+    for step in 0..900 {
+        edit(&mut s, &mut dice, &mut ids, album);
+        // (not every frame draws the albums: sometimes several edits pass between two looks)
+        if dice.roll(4) == 0 {
+            continue;
+        }
+        let moved = tally.revision() != s.catalog.revision;
+        match (moved, tally.update(&s.catalog)) {
+            (true, true) => followed += 1,
+            (true, false) => whole += 1,
+            // (an edit that was refused changed nothing: nothing to follow)
+            (false, _) => {}
+        }
+        assert_eq!(shown(&tally), fresh(&s), "after edit {step}");
+    }
+    assert!(followed > 300 && whole > 20, "followed {followed} times, counted whole {whole} times");
+    // a photo deleted: the album of chosen photos it is in counts one less, without every photo being asked
+    let in_album = s
+        .catalog
+        .album(lightcraft_catalog::AlbumId(album))
+        .and_then(|a| a.photos.iter().copied().find(|id| s.catalog.photo(*id).is_some_and(|p| !p.deleted)));
+    if let Some(id) = in_album {
+        let before = tally.counts()[&lightcraft_catalog::AlbumId(album)];
+        s.execute("library.select", &json!({"ids": [id.0]})).unwrap();
+        s.execute("photo.delete", &json!({})).unwrap();
+        assert!(tally.update(&s.catalog), "followed");
+        assert_eq!(tally.counts()[&lightcraft_catalog::AlbumId(album)], before - 1);
+    }
+    // three hundred photos rated at once: counted from every photo, and right
+    let many: Vec<u64> = s.visible_cloned().iter().take(300).map(|id| id.0).collect();
+    s.execute("library.source", &json!({"kind": "all"})).unwrap();
+    s.execute("library.filter", &json!({"rating": 0, "text": ""})).unwrap();
+    let many: Vec<u64> = if many.len() >= 300 { many } else { s.visible_cloned().iter().take(300).map(|id| id.0).collect() };
+    if many.len() > 256 {
+        s.execute("library.select", &json!({"ids": many})).unwrap();
+        s.execute("photo.rate", &json!({"rating": 5})).unwrap();
+        assert!(!tally.update(&s.catalog), "more photos than are followed one by one");
+        assert_eq!(shown(&tally), fresh(&s));
+    }
+}
@@ -1594,8 +1594,12 @@ pub struct Caches {
    counts: Option<(u64, LibraryCounts)>,
    date_groups: Option<(u64, std::sync::Arc<Vec<lightcraft_catalog::DateGroup>>)>,
    filter_values: Option<(u64, std::sync::Arc<FilterValues>)>,
-    album_counts: Option<(u64, std::sync::Arc<std::collections::HashMap<lightcraft_catalog::AlbumId, usize>>)>,
-    /// How often the album counts were recomputed (tests check that unchanged frames don't).
+    /// The albums' counts: the minute they are of (while a smart album asks about "the last N days"), the tally
+    /// that follows edits, and the counts as the panels take them.
+    album_counts:
+        Option<(String, lightcraft_catalog::tally::AlbumTally, std::sync::Arc<std::collections::HashMap<lightcraft_catalog::AlbumId, usize>>)>,
+    /// How often the album counts were made from every photo (tests check that unchanged frames, and edits to a
+    /// photo or a few, don't).
    pub album_count_scans: usize,
    /// What the open smart-album rule dialog shows besides the rules ([`Caches::rules_view`]).
    rules_view: Option<(u64, std::sync::Arc<RulesView>)>,
@@ -1947,21 +1951,37 @@ impl Caches {
    ) -> std::sync::Arc<std::collections::HashMap<lightcraft_catalog::AlbumId, usize>> {
        let relative = cat.albums().any(|a| a.smart.as_ref().is_some_and(|f| f.depends_on_now()));
        let minute = if relative { now.get(..16).unwrap_or(now) } else { "" };
-        let k = key_of((cat.revision, minute));
-        match &self.album_counts {
-            Some((h, v)) if *h == k => v.clone(),
-            _ => {
-                if relative {
-                    // "in the last N days" counts back from the session clock, as in the grid
-                    lightcraft_catalog::rules::set_now(Some(now.to_string()));
-                }
-                let v: std::sync::Arc<std::collections::HashMap<_, _>> =
-                    std::sync::Arc::new(cat.albums().map(|a| (a.id, cat.album_count(a.id))).collect());
-                self.album_count_scans += 1;
-                self.album_counts = Some((k, v.clone()));
-                v
-            }
+        // the counts of this minute, if there are any: an edit to a photo or a few is followed from them, a smart
+        // album asking only those photos again; anything more, or a new minute, is counted from every photo
+        let kept = self.album_counts.take().filter(|(of, _, _)| of == minute);
+        if let Some((_, tally, shown)) = &kept
+            && tally.revision() == cat.revision
+        {
+            let shown = shown.clone();
+            self.album_counts = kept;
+            return shown;
        }
+        if relative {
+            // "in the last N days" counts back from the session clock, as in the grid. (Within a minute an edit asks
+            // only the photos it touched, at this second; the others were asked earlier in the minute. That is as
+            // far out as the counts already were between edits, and the next minute counts them all again.)
+            lightcraft_catalog::rules::set_now(Some(now.to_string()));
+        }
+        let tally = match kept {
+            Some((_, mut tally, _)) => {
+                if !tally.update(cat) {
+                    self.album_count_scans += 1;
+                }
+                tally
+            }
+            None => {
+                self.album_count_scans += 1;
+                lightcraft_catalog::tally::AlbumTally::of(cat)
+            }
+        };
+        let shown = std::sync::Arc::new(tally.counts().clone());
+        self.album_counts = Some((minute.to_string(), tally, shown.clone()));
+        shown
    }
 }

@@ -2002,16 +2022,17 @@ mod cache_tests {
            c.album_counts(&cat, "2026-09-30T12:00:40");
        }
        assert_eq!(c.album_count_scans, 1, "unchanged frames in the same minute don't rescan");
-        // metadata change
+        // a photo rated: followed from that photo, not counted from every one
        cat.apply(Op::SetRating { id: PhotoId(2), rating: 4 }).unwrap();
        assert_eq!(c.album_counts(&cat, now)[&AlbumId(10)], 2);
+        assert_eq!(c.album_count_scans, 1);
        // rule change
        let rs: RuleSet = serde_json::from_value(serde_json::json!({"rules": [{"field": "rating", "op": "gte", "value": 5}]})).unwrap();
        cat.apply(Op::SetAlbumRules { id: AlbumId(10), rules: Box::new(Filter { rule_set: Some(rs), ..Default::default() }) }).unwrap();
        assert_eq!(c.album_counts(&cat, now)[&AlbumId(10)], 1);
        // an hour later the 11:59:30 photo has left "in the last hour", with no catalog change
        assert_eq!(c.album_counts(&cat, "2026-09-30T13:00:10")[&AlbumId(11)], 0);
-        assert_eq!(c.album_count_scans, 4);
+        assert_eq!(c.album_count_scans, 3, "new rules and a new minute are counted from every photo; the rating was not");
        lightcraft_catalog::rules::set_now(None);
    }

@@ -103,8 +103,12 @@
  keywords, dates, people and counts; a keyword leaves the folders, the dates and the counts as they are. With both
  side panels open, the frame after a keyword went from about 156 ms to about 54 ms in that library (2 ms with the
  panels closed, from 71 ms).
- Still tallied from every photo on each edit: the count beside each album and smart album in the left panel (about
-  25 ms for a smart album in that library), and the filter bar's lists of cameras, lenses and keywords.
+- The count beside each smart album in the left panel follows an edit the same way: the album asks only the photos
+  the edit touched, where it asked every photo of the library after each edit (about 25 ms an album in that library).
+  With that, the frame after a keyword is at about 29 ms there. New rules, a change to an album or the keyword list,
+  an edit to more than 256 photos, and each new minute for an album that asks about "the last N days" are still
+  counted from every photo.
+- Still tallied from every photo on each edit: the filter bar's lists of cameras, lenses and keywords.

 ### A view can hold its photos until you refresh it
 - View ▸ Hold View Until Refreshed (off unless you turn it on; kept with the library). With it on, an edit that makes