References: index source lookups during citation refresh (#4890)

Built-in citation refresh and first-cited bibliography ordering build temporary tag and
number indexes instead of scanning sources per field; duplicate-tag first matches are kept.

Rebased onto main by maintainers; the scaling test counts lookups instead of timing.

Co-Authored-By: OPUZZLES <109076338+OPUZZLES@users.noreply.github.com>
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FypgCYPZSei56X4AGFPSu5

crates/engine/src/cmd/citations/sources.rs

@@ -201,7 +201,18 @@ pub(super) fn bib_order(doc: &Document, st: &CiteStyle, lang: Option<String>) ->
     let mut idx: Vec<usize> = if cited.is_empty() {
         (0..doc.sources.len()).collect()
     } else if st.order == Order::Cited {
-        cited.iter().filter_map(|t| doc.sources.iter().position(|x| &x.tag == t)).collect()
+        let mut by_tag = std::collections::HashMap::new();
+        for (i, source) in doc.sources.iter().enumerate() {
+            wordcraft_doc::work::steps(1);
+            by_tag.entry(source.tag.as_str()).or_insert(i);
+        }
+        cited
+            .iter()
+            .filter_map(|t| {
+                wordcraft_doc::work::steps(1);
+                by_tag.get(t.as_str()).copied()
+            })
+            .collect()
     } else {
         let cited: std::collections::HashSet<&str> = cited.iter().map(String::as_str).collect();
         doc.sources.iter().enumerate().filter(|(_, x)| cited.contains(x.tag.as_str())).map(|(i, _)| i).collect()
@@ -233,6 +244,23 @@ pub fn update_citations(s: &mut Session) -> Result<(), CmdError> {
 fn refresh_citations(s: &mut Session) -> Result<(), CmdError> {
     let st = style(&s.bib_style);
     let order = bib_order(&s.doc, st, crate::cmd::default_lang(s));
+    // Native data can repeat tags: keep the first source record and, independently, the first
+    // bibliography position with that tag. Own the keys so paragraph writes don't borrow sources.
+    let fallback = order.len().saturating_add(1);
+    let mut by_tag = std::collections::HashMap::new();
+    for (i, source) in s.doc.sources.iter().enumerate() {
+        wordcraft_doc::work::steps(1);
+        by_tag.entry(source.tag.clone()).or_insert((i, fallback));
+    }
+    for (n, &i) in order.iter().enumerate() {
+        wordcraft_doc::work::steps(1);
+        if let Some(source) = s.doc.sources.get(i)
+            && let Some((_, number)) = by_tag.get_mut(&source.tag)
+            && *number == fallback
+        {
+            *number = n.saturating_add(1);
+        }
+    }
     for path in s.doc.para_paths(StoryRef::Body) {
         let Some(p) = s.doc.para(StoryRef::Body, &path) else { continue };
         let updates: Vec<(usize, String)> = p
@@ -242,8 +270,10 @@ fn refresh_citations(s: &mut Session) -> Result<(), CmdError> {
             .filter_map(|(k, o)| match o {
                 InlineObject::Field { instr, .. } => {
                     let (tag, pages) = citation_field(instr)?;
-                    let src = s.doc.sources.iter().find(|x| x.tag == tag)?;
-                    Some((k, cite(src, st.name, number_of(&s.doc, &order, tag), pages)))
+                    wordcraft_doc::work::steps(1);
+                    let &(i, number) = by_tag.get(tag)?;
+                    let src = s.doc.sources.get(i)?;
+                    Some((k, cite(src, st.name, number, pages)))
                 }
                 _ => None,
             })

crates/engine/src/cmd/citations/tests.rs

@@ -785,3 +785,100 @@ fn gost_numbers_follow_its_name_or_title_sort() {
     assert_eq!(find_style("Harvard - Anglia 2008").map(|x| x.name), Some("Harvard (Anglia)"));
     assert!(find_style("").is_none() && find_style("Vancouver").is_none());
 }
+
+fn lookup_session(tags: &[&str]) -> Session {
+    let mut p = Paragraph::new();
+    for tag in tags {
+        p.insert_object(
+            p.len(),
+            InlineObject::Field {
+                instr: format!("CITATION {tag}"),
+                result: "Original cache".into(),
+                locked: false,
+                dirty: true,
+                code: false,
+                form: String::new(),
+            },
+            &CharProps::default(),
+        )
+        .unwrap();
+    }
+    let mut doc = Document::new();
+    doc.body = vec![wordcraft_doc::para_block(p)];
+    Session::new(doc)
+}
+
+#[test]
+fn citation_source_lookups_keep_first_duplicates_case_and_style_order() {
+    let mut s = lookup_session(&["A", "B", "missing", "a", "B"]);
+    s.doc.sources = [
+        ("B", "Zed, Zo", "Zulu"),
+        ("A", "Able, Ann", "Alpha"),
+        ("B", "Baker, Ben", "Beta duplicate"),
+        ("a", "Middle, May", "Middle"),
+        ("unused", "Other, One", "Unused"),
+    ]
+    .into_iter()
+    .map(|(tag, author, title)| Source { tag: tag.into(), author: author.into(), title: title.into(), year: "2024".into(), ..Default::default() })
+    .collect();
+    let original = s.doc.clone();
+    for st in &STYLES {
+        s.bib_style = st.name.into();
+        let expected = if st.order == Order::Cited { vec![1, 0, 3] } else { vec![1, 2, 3, 0] };
+        assert_eq!(sources::bib_order(&s.doc, st, None), expected, "{}", st.name);
+        update_citations(&mut s).unwrap();
+        let p = s.doc.para(StoryRef::Body, &Path::top(0)).unwrap();
+        for (k, (source, number)) in [(1, 1), (0, 2), (usize::MAX, 0), (3, 3), (0, 2)].into_iter().enumerate() {
+            let InlineObject::Field { instr, result, locked, dirty, code, form } = &p.objects[k] else { panic!() };
+            let InlineObject::Field { instr: before, .. } = &original.para(StoryRef::Body, &Path::top(0)).unwrap().objects[k] else { panic!() };
+            assert_eq!(instr, before);
+            let expected = original.sources.get(source).map(|x| cite(x, st.name, number, "")).unwrap_or_else(|| "Original cache".into());
+            assert_eq!(result, &expected, "{}: {instr}", st.name);
+            assert!(!locked && *dirty && !code && form.is_empty());
+        }
+        assert_eq!(s.doc.sources, original.sources);
+    }
+    // Without citations even duplicate and unused source records remain in the bibliography.
+    s.doc.body.clear();
+    assert_eq!(sources::bib_order(&s.doc, style("IEEE"), None), vec![0, 1, 2, 3, 4]);
+}
+
+#[test]
+fn citation_refresh_source_lookup_work_scales_with_document_size() {
+    fn doc(n: usize) -> Session {
+        let mut p = Paragraph::new();
+        p.text = wordcraft_doc::para::OBJ.to_string().repeat(n);
+        p.runs = vec![wordcraft_doc::Run { len: p.text.len(), props: CharProps::default() }];
+        p.objects = (0..n)
+            .rev()
+            .map(|i| InlineObject::Field {
+                instr: format!("CITATION T{i}"),
+                result: String::new(),
+                locked: false,
+                dirty: false,
+                code: false,
+                form: String::new(),
+            })
+            .collect();
+        let mut d = Document::new();
+        d.body = vec![wordcraft_doc::para_block(p)];
+        d.sources = (0..n).map(|i| Source { tag: format!("T{i}"), ..Default::default() }).collect();
+        let mut s = Session::new(d);
+        s.bib_style = "IEEE".into();
+        s
+    }
+    // The source and bibliography-number lookups the refresh does (`wordcraft_doc::work`), not a
+    // timing: about 8x for 8x the citations and sources when each is indexed once, about 64x when
+    // every field scans the sources or the bibliography order again.
+    let (mut small, mut big) = (doc(2_000), doc(8 * 2_000));
+    let s1 = wordcraft_doc::work::count(|| update_citations(&mut small).unwrap());
+    let s8 = wordcraft_doc::work::count(|| update_citations(&mut big).unwrap());
+    let ratio = s8 as f64 / s1.max(1) as f64;
+    eprintln!("citation refresh steps: {s1} -> {s8}, {ratio:.1}x for 8x input");
+    // At least one counted step per source and per field: a revert to scans that don't count
+    // would leave only the paragraph walk's few steps.
+    assert!(s1 >= 2 * 2_000, "only {s1} steps counted, so the lookups this test watches no longer count them (wordcraft_doc::work)");
+    assert!(ratio <= 12.0, "citation refresh repeats source lookups: {ratio:.1}x the steps for 8x input ({s1} -> {s8})");
+    let p = big.doc.para(StoryRef::Body, &Path::top(0)).unwrap();
+    assert!(p.objects.iter().enumerate().all(|(i, o)| matches!(o, InlineObject::Field { result, .. } if result == &format!("[{}]", i + 1))));
+}