Merge #2151 (9fdf03cf, with #465 at 3f8e6c28 and main at 6ead1fb5, where #462 is merged) into #2144

# Conflicts:
#	crates/engine/src/lib.rs
#	docs/macros.md

crates/engine/src/cmd/data.rs

@@ -652,6 +652,24 @@ fn filter_blank(v: &Value) -> bool {
     v.is_empty() || matches!(v, Value::Text(t) if t.is_empty())
 }
 
+/// Excel grows an AutoFilter range when a value is entered in the row right below it, in one of its columns (#2342).
+/// A table's own filter is left to the table, and a row typed after a blank row does not count.
+/// A formula counts before it is calculated: this runs inside the edit, ahead of the recalc.
+pub(crate) fn grow_autofilter(sh: &mut Sheet, at: CellRef) {
+    if !sh.cells.get(at).is_some_and(|c| !filter_blank(&c.value) || c.formula.is_some()) {
+        return;
+    }
+    let Some(af) = sh.autofilter.as_ref() else { return };
+    let r = af.range;
+    if r.end.row.checked_add(1) != Some(at.row) || at.col < r.start.col || at.col > r.end.col || sh.tables.iter().any(|t| t.range == r) {
+        return;
+    }
+    let grown = RangeRef::new(r.start, CellRef::new(at.row, r.end.col));
+    if let Some(af) = sh.autofilter.as_mut() {
+        af.range = grown;
+    }
+}
+
 /// Hides rows of the autofilter range that fail any criterion.
 pub(crate) fn apply_filter(wb: &mut Workbook, sheet: usize) {
     let Some(sh) = wb.sheet(sheet) else { return };
@@ -1554,6 +1572,33 @@ mod tests {
         }
     }
 
+    #[test]
+    fn autofilter_range_grows_when_a_value_is_entered_in_the_row_below() {
+        let mut s = Session::new();
+        s.new_workbook();
+        s.execute("range.setValues", json!({"range": "A1", "values": [["Name", "Qty"], ["d", 4], ["b", 2]]})).unwrap();
+        s.execute("data.filter", json!({"range": "A1:B3"})).unwrap();
+        let range = |s: &Session| s.doc().unwrap().wb.active().unwrap().autofilter.as_ref().map(|a| a.range.a1());
+        s.execute("cell.set", json!({"cell": "A4", "input": "e"})).unwrap();
+        assert_eq!(range(&s), Some("A1:B4".to_string()));
+        s.execute("range.setValues", json!({"range": "A5", "values": [["f", 6]]})).unwrap();
+        assert_eq!(range(&s), Some("A1:B5".to_string()));
+        // A row typed after a blank row does not grow it.
+        s.execute("cell.set", json!({"cell": "A7", "input": "x"})).unwrap();
+        assert_eq!(range(&s), Some("A1:B5".to_string()));
+        // A cell outside the filter's columns does not either.
+        s.execute("cell.set", json!({"cell": "C6", "input": "x"})).unwrap();
+        assert_eq!(range(&s), Some("A1:B5".to_string()));
+        // Empty text is a blank entry, not a value.
+        s.execute("range.setValues", json!({"range": "A6", "parse": false, "values": [[""]]})).unwrap();
+        assert_eq!(range(&s), Some("A1:B5".to_string()));
+        // A formula and Ctrl+Enter (range.fill) are entries too.
+        s.execute("cell.set", json!({"cell": "B6", "input": "=1+1"})).unwrap();
+        assert_eq!(range(&s), Some("A1:B6".to_string()));
+        s.execute("range.fill", json!({"range": "A7:B7", "input": "g"})).unwrap();
+        assert_eq!(range(&s), Some("A1:B7".to_string()));
+    }
+
     #[test]
     fn list_validation_without_formula1_is_an_error() {
         let mut s = Session::new();

crates/engine/src/cmd/edit.rs

@@ -272,6 +272,7 @@ fn cell_set(s: &mut Session, p: &Json) -> Result<Json> {
                 sh.remove_cell(at);
             }
         }
+        super::data::grow_autofilter(sh, at);
         cx.touch(sheet, at);
         Ok(Json::Null)
     })?;
@@ -350,6 +351,7 @@ fn range_set_values(s: &mut Session, p: &Json) -> Result<Json> {
                         sh.remove_cell(at);
                     }
                 }
+                super::data::grow_autofilter(sh, at);
                 cx.touch(sheet, at);
                 count += 1;
             }
@@ -390,7 +392,9 @@ fn range_fill(s: &mut Session, p: &Json) -> Result<Json> {
                     let shifted = gridcraft_formula::adjust::shift_relative(e, at.row as i64 - origin.row as i64, at.col as i64 - origin.col as i64);
                     cell.formula = Some(std::sync::Arc::new(Formula::from_expr(shifted)));
                 }
-                cx.sheet_mut(sheet)?.set_cell(at, cell);
+                let sh = cx.sheet_mut(sheet)?;
+                sh.set_cell(at, cell);
+                super::data::grow_autofilter(sh, at);
                 cx.touch(sheet, at);
             }
         }

crates/engine/src/cmd/extra.rs

@@ -212,6 +212,15 @@ pub fn specs() -> Vec<CommandSpec> {
             trust_scripts
         ),
         cmd!("automate.deleteScript", "Delete Script", [], None, "{name}", has_doc, delete_script),
+        cmd!(
+            noundo "automate.stopAction",
+            "Stop Action",
+            [],
+            None,
+            "{} → {stopped: bool}: stops an action left running on the active workbook (an open undo group, such as a debugger's) and puts back the workbook and its undo history as they were when it began",
+            has_doc,
+            stop_action
+        ),
         // Cell controls, themes, borders, styles
         cmd!(
             "insert.checkbox",
@@ -800,10 +809,9 @@ fn split(s: &mut Session, _: &Json) -> Result<Json> {
 
 fn new_window(s: &mut Session, _: &Json) -> Result<Json> {
     let d = s.doc()?.clone();
-    let mut copy = crate::DocState::new((*d.wb).clone(), d.path.clone(), format!("{}:2", d.display_title()));
+    let mut copy = crate::DocState::new((*d.wb).clone(), d.path.clone(), format!("{}:2", d.display_title()), d.trust);
     copy.selection = d.selection.clone();
     copy.scripts_trusted = d.scripts_trusted;
-    copy.trust = d.trust;
     let i = s.add_document(copy);
     Ok(json!({"index": i}))
 }
@@ -1088,6 +1096,18 @@ fn run_script(s: &mut Session, p: &Json) -> Result<Json> {
     Ok(json!({"ran": n}))
 }
 
+/// Automate › Stop Action: rolls back the undo groups open on the active workbook, so Undo and
+/// Redo work there again.
+fn stop_action(s: &mut Session, _: &Json) -> Result<Json> {
+    // Inside another command the open groups are that command's own (an action in progress):
+    // a step of it that stopped them would pull the workbook from under the rest of its run.
+    if s.depth() > 1 {
+        return Err(EngineError::Disabled("automate.stopAction".into(), "Stop Action runs on its own, not as a step of a script or macro.".into()));
+    }
+    let uid = s.doc()?.uid;
+    Ok(json!({"stopped": s.abort_groups(uid)}))
+}
+
 fn delete_script(s: &mut Session, p: &Json) -> Result<Json> {
     let name = str_param(p, "name").ok_or_else(|| bad("automate.deleteScript", "missing `name`"))?.to_string();
     edit(s, |cx| {

crates/engine/src/cmd/file.rs

@@ -34,7 +34,7 @@ fn new_workbook(s: &mut Session, p: &Json) -> Result<Json> {
         Some(name) => {
             let wb = crate::sample::build(name).ok_or_else(|| bad("file.new", format!("unknown sample `{name}`")))?;
             let title = crate::sample::title(name);
-            s.add_document(DocState::new(wb, None, title))
+            s.add_document(DocState::new(wb, None, title, crate::Trust::Local))
         }
         None => s.new_workbook(),
     };
@@ -83,12 +83,11 @@ fn open(s: &mut Session, p: &Json) -> Result<Json> {
     {
         s.close_document(0);
     }
-    let mut doc = DocState::new(wb, path, title);
+    // Someone else may have made the file: its active content asks before it runs.
+    let mut doc = DocState::new(wb, path, title, crate::Trust::FromFile);
     if crate::io::FileKind::from_path(&name).is_none() && gridcraft_xlsx::sniff(&bytes) == gridcraft_xlsx::Format::Xlsx {
         doc.content_kind = Some(crate::io::FileKind::Xlsx);
     }
-    // Someone else may have made the file: its active content asks before it runs.
-    doc.trust = crate::Trust::FromFile;
     let i = s.add_document(doc);
     // The decrypted-workbook notice is for the info bar, not a message box.
     let shown: Vec<&str> = warnings.iter().map(String::as_str).filter(|w| *w != crate::io::DECRYPTED_NOTICE).collect();

crates/engine/src/cmd/formulas.rs

@@ -1,7 +1,7 @@
 //! Formulas tab: AutoSum, Insert Function, names, auditing, calculation.
 
 use gridcraft_core::{CellError, CellRef, RangeRef, Value};
-use gridcraft_model::{CalcMode, DefinedName, Formula};
+use gridcraft_model::{CalcMode, DefinedName, Formula, Workbook};
 use serde_json::{Value as Json, json};
 
 use super::*;
@@ -32,18 +32,26 @@ pub fn specs() -> Vec<CommandSpec> {
             "Define Name…",
             ["Formulas", "Defined Names"],
             None,
-            "{name, refersTo?: \"=Sheet1!$A$1:$A$10\" (default: the selection), scope?: \"Workbook\"|sheet name, comment?, oldName?, oldScope?} (oldName: edit that name (Name Manager › Edit); a new name also renames it in formulas)",
+            "{name, refersTo?: \"=Sheet1!$A$1:$A$10\" (default: the selection), scope?: \"Workbook\"|sheet name, comment?, oldName?, oldScope?, force?: bool} (oldName: edit that name (Name Manager › Edit); a new name also renames it in formulas; a hidden name GridCraft keeps its own data in, `_gc_<key>_<n>` or `_sc_<key>`, is refused unless force is true)",
             has_doc,
             define_name
         ),
-        cmd!("formulas.deleteName", "Delete Name", [], None, "{name, scope?: \"Workbook\"|sheet name (default: every scope)}", has_doc, delete_name),
+        cmd!(
+            "formulas.deleteName",
+            "Delete Name",
+            [],
+            None,
+            "{name, scope?: \"Workbook\"|sheet name (default: every scope), force?: bool} (a hidden name GridCraft keeps its own data in, `_gc_<key>_<n>` or `_sc_<key>`, is refused unless force is true)",
+            has_doc,
+            delete_name
+        ),
         cmd!(query "formulas.nameManager", "Name Manager", ["Formulas", "Defined Names"], Some("Cmd+F3"), "{} → names with values", has_doc, name_manager),
         cmd!(
             "formulas.createFromSelection",
             "Create from Selection",
             ["Formulas", "Defined Names"],
             Some("Cmd+Shift+F3"),
-            "{range?, top?: true, left?: false, bottom?: false, right?: false, replace?: false (existing names are kept and listed in `skipped`)}",
+            "{range?, top?: true, left?: false, bottom?: false, right?: false, replace?: false (existing names are kept and listed in `skipped`), force?: false (labels of the shape GridCraft keeps its own data in, `_gc_<key>_<n>` and `_sc_<key>`, are listed in `skipped` unless force is true)}",
             has_doc,
             create_from_selection
         ),
@@ -268,6 +276,9 @@ fn define_name(s: &mut Session, p: &Json) -> Result<Json> {
     if !valid_name(&name) {
         return Err(EngineError::Other("The name that you entered is not valid.".into()));
     }
+    for n in [Some(name.as_str()), str_param(p, "oldName")].into_iter().flatten() {
+        refuse_stash_name(&s.doc()?.wb, p, n)?;
+    }
     let d = s.doc()?;
     let scope = match str_param(p, "scope") {
         None | Some("Workbook") => None,
@@ -374,8 +385,29 @@ fn rename_name_in_formulas(wb: &mut gridcraft_model::Workbook, old: &str, old_sc
     }
 }
 
+/// Refuses a name of the shape GridCraft keeps its own workbook data in ([`super::stash`]):
+/// defining, renaming or deleting one would replace or remove part of an item (a stored macro,
+/// script or view) and leave the rest behind. `force: true` lets the call go ahead.
+/// Whether `name` is one the workbook holds GridCraft's own data in: a hidden workbook-scope
+/// name of an item's shape. A visible name of that shape is the person's own (see `stash`), and
+/// the name commands may change it.
+fn holds_gridcraft_data(wb: &Workbook, name: &str) -> bool {
+    // The stash's own rule, so the two never disagree: a legacy `_sc_` item is read visible or not.
+    stash::holds_item(wb, name)
+}
+
+fn refuse_stash_name(wb: &Workbook, p: &Json, name: &str) -> Result<()> {
+    if holds_gridcraft_data(wb, name) && !bool_param(p, "force").unwrap_or(false) {
+        return Err(EngineError::Other(format!(
+            "'{name}' is a name GridCraft keeps its own data in (stored macros, scripts or views). Pass `force: true` to change it anyway."
+        )));
+    }
+    Ok(())
+}
+
 fn delete_name(s: &mut Session, p: &Json) -> Result<Json> {
     let name = str_param(p, "name").ok_or_else(|| bad("formulas.deleteName", "missing `name`"))?.to_string();
+    refuse_stash_name(&s.doc()?.wb, p, &name)?;
     let scope = match str_param(p, "scope") {
         None => None,
         Some("Workbook") => Some(None),
@@ -461,7 +493,11 @@ fn create_from_selection(s: &mut Session, p: &Json) -> Result<Json> {
     // Excel asks before replacing an existing name; here it's kept unless `replace` is set, and
     // reported.
     let exists = |n: &str| d.wb.names.iter().any(|x| x.name.eq_ignore_ascii_case(n) && x.scope.is_none());
-    let (made, skipped): (Vec<_>, Vec<_>) = made.into_iter().partition(|(n, _)| replace || !exists(n));
+    // A label that names one of GridCraft's own hidden data names would replace part of an item
+    // (see `refuse_stash_name`).
+    let force = bool_param(p, "force").unwrap_or(false);
+    let ours = |n: &str| !force && holds_gridcraft_data(&d.wb, n);
+    let (made, skipped): (Vec<_>, Vec<_>) = made.into_iter().partition(|(n, _)| !ours(n) && (replace || !exists(n)));
     let skipped: Vec<String> = skipped.into_iter().map(|(n, _)| n).collect();
     let n = made.len();
     edit(s, |cx| {
@@ -742,6 +778,57 @@ mod tests {
         assert_eq!(r["refersTo"], "=Sheet1!$A$1");
     }
 
+    #[test]
+    fn the_name_commands_leave_gridcraft_data_alone() {
+        // Defining, renaming or deleting a name of a stored item's shape would replace or remove
+        // one of its chunks and leave the rest behind: refused unless `force` is true.
+        let mut s = Session::new();
+        s.new_workbook();
+        let wb = std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().wb);
+        let text = "m".repeat(9_000);
+        stash::set(wb, "macro_1", &text).unwrap();
+        wb.names.push(DefinedName { name: "_sc_watch".into(), scope: None, formula: "\"[]\"".into(), comment: String::new(), hidden: true });
+        // A legacy item is read whether or not it is hidden, so a visible one is data too.
+        wb.names.push(DefinedName { name: "_sc_old".into(), scope: None, formula: "\"[]\"".into(), comment: String::new(), hidden: false });
+        let names = |s: &Session| s.doc().unwrap().wb.names.clone();
+        let before = names(&s);
+        assert!(before.len() >= 3, "the item takes two chunks");
+        for params in [
+            json!({"name": "_gc_macro_1_0", "refersTo": "=1"}),
+            json!({"name": "_SC_Watch", "refersTo": "=1"}),
+            json!({"name": "Mine", "refersTo": "=1", "oldName": "_gc_macro_1_1"}),
+            json!({"name": "_gc_macro_1_0", "refersTo": "=1", "force": false}),
+            json!({"name": "Mine2", "refersTo": "=1", "oldName": "_SC_old"}),
+        ] {
+            let err = s.execute("formulas.defineName", params.clone()).unwrap_err().to_string();
+            assert!(err.contains("force"), "{params}: {err}");
+        }
+        for name in ["_gc_macro_1_1", "_sc_watch", "_sc_old"] {
+            let err = s.execute("formulas.deleteName", json!({"name": name})).unwrap_err().to_string();
+            assert!(err.contains("force"), "{name}: {err}");
+        }
+        s.execute("range.setValues", json!({"range": "A1:B2", "values": [["_gc_macro_1_0", "Rate"], [1, 2]]})).unwrap();
+        let r = s.execute("formulas.createFromSelection", json!({"range": "A1:B2", "top": true, "replace": true})).unwrap();
+        assert_eq!((r["created"].clone(), r["skipped"].clone()), (json!(1), json!(["_gc_macro_1_0"])));
+        let after = names(&s);
+        assert_eq!(after.iter().filter(|n| n.name != "Rate").cloned().collect::<Vec<_>>(), before);
+        assert_eq!(stash::get(&s.doc().unwrap().wb, "macro_1").as_deref(), Some(text.as_str()));
+        // A visible name of the shape is the person's own (the stash never reads it): the name
+        // commands change and remove it without `force`, as Name Manager cannot pass one.
+        let own = DefinedName { name: "_gc_rate_0".into(), scope: None, formula: "=1".into(), comment: String::new(), hidden: false };
+        std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().wb).names.push(own.clone());
+        s.execute("formulas.defineName", json!({"name": "Rate0", "refersTo": "=2", "oldName": "_gc_rate_0"})).unwrap();
+        s.execute("formulas.deleteName", json!({"name": "Rate0"})).unwrap();
+        std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().wb).names.push(own);
+        s.execute("formulas.deleteName", json!({"name": "_gc_rate_0"})).unwrap();
+        assert_eq!(stash::get(&s.doc().unwrap().wb, "macro_1").as_deref(), Some(text.as_str()), "the hidden item is untouched");
+        // Ordinary names and forced calls go ahead.
+        s.execute("formulas.deleteName", json!({"name": "Rate"})).unwrap();
+        s.execute("formulas.defineName", json!({"name": "_gc_macro_1_1", "refersTo": "=1", "force": true})).unwrap();
+        s.execute("formulas.deleteName", json!({"name": "_gc_macro_1_1", "force": true})).unwrap();
+        assert!(!names(&s).iter().any(|n| n.name == "_gc_macro_1_1"));
+    }
+
     #[test]
     fn arabic_defined_names_count_characters_not_bytes() {
         let mut s = Session::new();

crates/engine/src/cmd/mod.rs

@@ -109,7 +109,7 @@ pub fn can_redo(s: &Session) -> std::result::Result<(), String> {
 /// Undo and Redo wait while an undo group is open on the active document (an action that spans
 /// several calls is still running).
 fn not_grouping(s: &Session) -> std::result::Result<(), String> {
-    if s.active().is_some_and(|d| s.in_group(d.uid)) { Err("finish or stop the running action first".into()) } else { Ok(()) }
+    if s.active().is_some_and(|d| s.in_group(d.uid)) { Err("finish or stop the running action first (automate.stopAction)".into()) } else { Ok(()) }
 }
 pub fn has_clipboard(s: &Session) -> std::result::Result<(), String> {
     has_doc(s)?;

crates/engine/src/cmd/pivot.rs

@@ -179,6 +179,15 @@ pub fn specs() -> Vec<CommandSpec> {
             has_doc,
             change_source
         ),
+        cmd!(
+            "pivot.showDetails",
+            "Show Details",
+            [],
+            None,
+            "{cell?: \"B5\" (a value, subtotal or grand total cell of a PivotTable on the active sheet; default: the active cell)} puts the source records behind that value on a new sheet before the current one → {sheet, records}",
+            has_doc,
+            show_details
+        ),
         cmd!(
             "pivot.delete",
             "Delete PivotTable",
@@ -654,16 +663,7 @@ fn insert_pivot(s: &mut Session, p: &Json) -> Result<Json> {
         let (si, anchor) = match target {
             Some(t) => t,
             None => {
-                let at = cx.wb.active_sheet.min(cx.wb.sheets.len());
-                let nm = cx.wb.next_sheet_name();
-                cx.wb.sheets.insert(at, Arc::new(Sheet::new(nm)));
-                for n in cx.wb.names.iter_mut() {
-                    if let Some(sc) = n.scope.as_mut()
-                        && *sc >= at
-                    {
-                        *sc += 1;
-                    }
-                }
+                let at = insert_sheet_before_active(cx);
                 new_sheet_at = Some(at);
                 (at, CellRef::new(2, 0))
             }
@@ -726,6 +726,104 @@ fn insert_pivot(s: &mut Session, p: &Json) -> Result<Json> {
     Ok(res)
 }
 
+/// Inserts a new, empty sheet before the active one (Excel's place for a new PivotTable or a
+/// Show Details sheet); returns its index.
+fn insert_sheet_before_active(cx: &mut Ctx) -> usize {
+    let at = cx.wb.active_sheet.min(cx.wb.sheets.len());
+    let nm = cx.wb.next_sheet_name();
+    cx.wb.sheets.insert(at, Arc::new(Sheet::new(nm)));
+    for n in cx.wb.names.iter_mut() {
+        if let Some(sc) = n.scope.as_mut()
+            && *sc >= at
+        {
+            *sc += 1;
+        }
+    }
+    at
+}
+
+/// Show Details (double-click a value cell): the source records behind it, as values with their
+/// column's number format, under the source headers on a new sheet.
+fn show_details(s: &mut Session, p: &Json) -> Result<Json> {
+    let d = s.doc()?;
+    let at = cell_param(s, p, "cell")?.unwrap_or(d.selection.active);
+    let active = d.wb.active_sheet;
+    let sh = d.wb.active().ok_or(EngineError::NoDocument)?;
+    let pt = sh
+        .pivots
+        .iter()
+        .find(|pt| pt.last_range.is_some_and(|r| r.contains(at)))
+        .ok_or_else(|| EngineError::Disabled("pivot.showDetails".into(), pv::NOT_A_VALUE_CELL.into()))?;
+    let origin = pt.last_range.map(|r| r.start).unwrap_or_default();
+    let found = pv::detail_records(&d.wb, pt, active, at.row.saturating_sub(origin.row), at.col.saturating_sub(origin.col)).map_err(other)?;
+    let (src, records) = found.ok_or_else(|| EngineError::Disabled("pivot.showDetails".into(), pv::NOT_A_VALUE_CELL.into()))?;
+    if d.wb.protected_structure {
+        return Err(other("The workbook is protected and cannot be changed."));
+    }
+    let count = records.len();
+    let mut new_sheet_at: Option<usize> = None;
+    let res = edit(s, |cx| {
+        let si = insert_sheet_before_active(cx);
+        new_sheet_at = Some(si);
+        let base = cx.wb.styles.get(StyleId::default()).clone();
+        let fmts: Vec<StyleId> = src
+            .formats
+            .iter()
+            .map(|f| {
+                let mut st = base.clone();
+                st.num_fmt = NumFmt::new(f);
+                cx.wb.styles.intern(st)
+            })
+            .collect();
+        // Columns wide enough for their text, as the PivotTable report's are (dates would show
+        // as ####); the first rows are enough to judge a column.
+        let font = base.font.clone();
+        let widths: Vec<f32> = src
+            .headers
+            .iter()
+            .enumerate()
+            .map(|(c, h)| {
+                let fmt = src.formats.get(c).map(String::as_str).unwrap_or("General");
+                let texts = records.iter().take(1_000).filter_map(|i| src.rows.get(*i).and_then(|r| r.get(c)));
+                let widest = texts
+                    .map(|v| crate::cmd::format::approx_text_width(&crate::display::format(v, fmt, &cx.wb).text, font.size, false))
+                    .fold(crate::cmd::format::approx_text_width(h, font.size, false), f32::max);
+                (widest + 14.0).min(320.0)
+            })
+            .collect();
+        let sh = cx.sheet_mut(si)?;
+        for (c, w) in widths.into_iter().enumerate() {
+            if w > sh.col_width(c as u32) {
+                sh.cols.entry(c as u32).or_default().size = Some(w);
+            }
+        }
+        for (c, h) in src.headers.iter().enumerate() {
+            sh.set_cell(CellRef::new(0, c as u32), Cell { value: gridcraft_core::Value::text(h.as_str()), formula: None, style: StyleId::default() });
+        }
+        for (r, i) in records.iter().enumerate() {
+            let Some(rec) = src.rows.get(*i) else { continue };
+            for (c, v) in rec.iter().enumerate() {
+                let style = fmts.get(c).copied().unwrap_or_default();
+                if v.is_empty() && style == StyleId::default() {
+                    continue;
+                }
+                sh.set_cell(CellRef::new(r as u32 + 1, c as u32), Cell { value: v.clone(), formula: None, style });
+            }
+        }
+        let name = sh.name.clone();
+        cx.wb.active_sheet = si;
+        *cx.sel = crate::Selection::at(CellRef::new(0, 0));
+        cx.structural = true;
+        Ok(json!({"sheet": name, "records": count}))
+    })?;
+    if let Some(at) = new_sheet_at
+        && let Ok(d) = s.doc_mut()
+    {
+        d.sheet_selections.insert(at.min(d.sheet_selections.len()), crate::Selection::default());
+    }
+    Ok(res)
+}
+
 fn summarize_table(s: &mut Session, p: &Json) -> Result<Json> {
     let d = s.doc()?;
     let name = match str_param(p, "table") {
@@ -2236,6 +2334,7 @@ mod tests {
             "data.refreshAll",
             "pivot.changeSource",
             "pivot.delete",
+            "pivot.showDetails",
         ];
         let hostile = [
             json!({}),
@@ -2363,4 +2462,122 @@ mod tests {
         assert_eq!(l[0]["anchor"], "A3");
         s.execute("pivot.refresh", json!({"pivot": "PivotTable1"})).unwrap();
     }
+
+    /// The active sheet's first five columns as shown, down to the first blank row.
+    fn sheet_rows(s: &mut Session) -> Vec<Vec<String>> {
+        let mut out = Vec::new();
+        for r in 0..30u32 {
+            let row: Vec<String> = (0..5)
+                .map(|c| s.execute("cell.get", json!({"cell": CellRef::new(r, c).a1()})).unwrap()["text"].as_str().unwrap_or("").to_string())
+                .collect();
+            if row.iter().all(String::is_empty) {
+                break;
+            }
+            out.push(row);
+        }
+        out
+    }
+
+    #[test]
+    fn show_details_lists_a_value_cells_records_on_a_new_sheet() {
+        let mut s = sales();
+        s.execute("insert.pivotTable", json!({"rows": ["Region"], "columns": ["Product"], "values": ["Sales"]})).unwrap();
+        // B5 is East × Apples (100 + 60); the active cell is the default.
+        s.execute("selection.set", json!({"range": "B5"})).unwrap();
+        let r = s.execute("pivot.showDetails", json!({})).unwrap();
+        assert_eq!(r["records"], 2);
+        let d = s.doc().unwrap();
+        assert_eq!(d.wb.sheets.len(), 3);
+        assert_eq!(d.wb.active_sheet, 0, "the new sheet goes before the PivotTable's");
+        assert_eq!(d.wb.active().unwrap().name, "Sheet3");
+        assert_eq!(
+            sheet_rows(&mut s),
+            vec![
+                vec!["Region", "Product", "Date", "Units", "Sales"],
+                vec!["East", "Apples", "1/15/2026", "10", "100"],
+                vec!["East", "Apples", "9/9/2026", "6", "60"],
+            ]
+        );
+        // The dates fit their column instead of showing ####.
+        let sh = s.doc().unwrap().wb.active().unwrap().clone();
+        assert!(sh.col_width(2) > sh.col_width(5), "{} vs {}", sh.col_width(2), sh.col_width(5));
+        // One undo step takes the sheet away.
+        s.execute("edit.undo", json!({})).unwrap();
+        assert_eq!(s.doc().unwrap().wb.sheets.len(), 2);
+        assert_eq!(val(&s, "B5"), Value::Number(160.0));
+    }
+
+    #[test]
+    fn show_details_on_a_total_takes_every_record_it_adds_up() {
+        let mut s = sales();
+        s.execute("insert.pivotTable", json!({"rows": ["Region"], "columns": ["Product"], "values": ["Sales"]})).unwrap();
+        let records = |s: &mut Session, cell: &str| {
+            let n = s.execute("pivot.showDetails", json!({"cell": cell})).unwrap()["records"].as_u64().unwrap();
+            s.execute("edit.undo", json!({})).unwrap();
+            n
+        };
+        assert_eq!(records(&mut s, "E5"), 5, "East's row total");
+        assert_eq!(records(&mut s, "C8"), 4, "Pears' column total");
+        assert_eq!(records(&mut s, "E8"), 12, "the grand total");
+        // A subtotal at the top of its group, in a two-level row axis.
+        s.execute("pivot.moveField", json!({"field": "Product", "from": "columns", "to": "rows"})).unwrap();
+        assert_eq!(text(&s, "A4"), "East");
+        assert_eq!(records(&mut s, "B4"), 5);
+        assert_eq!(text(&s, "A6"), "Pears");
+        assert_eq!(records(&mut s, "B6"), 2);
+    }
+
+    #[test]
+    fn show_details_leaves_out_the_records_the_report_filters_out() {
+        let mut s = sales();
+        s.execute("insert.pivotTable", json!({"rows": ["Product"], "values": ["Sales"], "filters": ["Region"]})).unwrap();
+        s.execute("pivot.filter", json!({"field": "Region", "selected": ["East"]})).unwrap();
+        s.execute("pivot.filter", json!({"field": "Product", "selected": ["Apples"]})).unwrap();
+        let gt = find_row(&s, "Grand Total").unwrap();
+        let r = s.execute("pivot.showDetails", json!({"cell": CellRef::new(gt, 1).a1()})).unwrap();
+        assert_eq!(r["records"], 2);
+        let rows = sheet_rows(&mut s);
+        assert_eq!(rows.len(), 3);
+        assert!(rows.iter().skip(1).all(|row| row[0] == "East" && row[1] == "Apples"), "{rows:?}");
+    }
+
+    #[test]
+    fn show_details_refuses_a_report_its_source_has_moved_on_from() {
+        let mut s = sales();
+        s.execute("insert.pivotTable", json!({"rows": ["Region"], "columns": ["Product"], "values": ["Sales"]})).unwrap();
+        // B5 still shows East × Apples (160), but the first East record is now Central's.
+        s.execute("range.setValues", json!({"range": "Sheet1!A2", "values": [["Central"]]})).unwrap();
+        assert_eq!(val(&s, "B5"), Value::Number(160.0));
+        let e = s.execute("pivot.showDetails", json!({"cell": "B5"})).unwrap_err().to_string();
+        assert!(e.contains("out of date"), "{e}");
+        assert_eq!(s.doc().unwrap().wb.sheets.len(), 2);
+        // After a refresh East × Apples (now B6) has the one record left.
+        s.execute("pivot.refresh", json!({})).unwrap();
+        assert_eq!(text(&s, "A6"), "East");
+        assert_eq!(s.execute("pivot.showDetails", json!({"cell": "B6"})).unwrap()["records"], 1);
+    }
+
+    #[test]
+    fn show_details_refuses_a_report_that_would_now_be_shorter() {
+        let mut s = sales();
+        s.execute("insert.pivotTable", json!({"rows": ["Region"], "columns": ["Product"], "values": ["Sales"]})).unwrap();
+        // North goes away: the source now gives one row fewer, so row 8 is no longer the Grand Total.
+        s.execute("range.setValues", json!({"range": "Sheet1!A5", "values": [["East"]]})).unwrap();
+        s.execute("range.setValues", json!({"range": "Sheet1!A8", "values": [["East"]]})).unwrap();
+        s.execute("range.setValues", json!({"range": "Sheet1!A11", "values": [["East"]]})).unwrap();
+        assert_eq!(text(&s, "A8"), "Grand Total");
+        let e = s.execute("pivot.showDetails", json!({"cell": "E8"})).unwrap_err().to_string();
+        assert!(e.contains("out of date"), "{e}");
+        assert_eq!(s.doc().unwrap().wb.sheets.len(), 2);
+    }
+
+    #[test]
+    fn show_details_needs_a_value_cell() {
+        let mut s = sales();
+        s.execute("insert.pivotTable", json!({"rows": ["Region"], "columns": ["Product"], "values": ["Sales"]})).unwrap();
+        for cell in ["A3", "B3", "A4", "B4", "A5", "A8", "H20"] {
+            assert!(s.execute("pivot.showDetails", json!({"cell": cell})).is_err(), "{cell}");
+        }
+        assert_eq!(s.doc().unwrap().wb.sheets.len(), 2);
+    }
 }

crates/engine/src/cmd/stash.rs

@@ -7,8 +7,16 @@
 //! one that Excel accepts. Older items (custom views, scripts, the watch list and scenarios) keep
 //! one name each, `_sc_<key>`; reading accepts that form as well, so those items can move to this
 //! format later, and a write of the same key replaces it.
+//!
+//! XML can't carry every character (controls other than tab, line feed and carriage return, and
+//! U+FFFE and U+FFFF), and the xlsx writer drops those. In `_gc_` items each of them, and the
+//! marker itself, is written as [`ESCAPE`] followed by its code as four upper-case hex digits
+//! (`U+0001` becomes `\u{FDD0}0001`), and reading turns them back. The marker is a Unicode
+//! noncharacter, which XML carries but real text does not use, so the escape costs nothing on
+//! ordinary text. Legacy `_sc_` items are read as they are.
 
 use std::collections::{BTreeMap, HashSet};
+use std::fmt::Write as _;
 
 use gridcraft_model::{DefinedName, Workbook};
 use serde_json::Value as Json;
@@ -25,23 +33,106 @@ const FORMULA_UNITS: usize = 8_192;
 const MAX_KEY: usize = 200;
 /// The longest item: 4,194,304 characters.
 pub const MAX_ITEM_CHARS: usize = 4 * 1024 * 1024;
+/// The most bytes the stored (escaped) form of an item can take: an escaped one-byte character
+/// takes seven (the three-byte marker and four digits).
+const MAX_STORED_BYTES: usize = MAX_ITEM_CHARS * 7;
+/// Starts an escaped character in a `_gc_` item: U+FDD0, a noncharacter.
+pub const ESCAPE: char = '\u{FDD0}';
+/// The most items [`items`] and [`keys`] return: the first this many keys in sorted order. A
+/// file can carry any number of hidden names; the rest are not listed.
+pub const MAX_ITEMS: usize = 10_000;
+
+/// The names that hold one item: its `_gc_` chunks by number (the first of each number) and its
+/// legacy `_sc_` name.
+#[derive(Default)]
+struct Parts<'a> {
+    chunks: BTreeMap<u64, &'a str>,
+    legacy: Option<&'a str>,
+}
+
+impl<'a> Parts<'a> {
+    fn add(&mut self, chunk: Option<u64>, formula: &'a str) {
+        match chunk {
+            Some(i) => {
+                self.chunks.entry(i).or_insert(formula);
+            }
+            None => self.legacy = self.legacy.or(Some(formula)),
+        }
+    }
+
+    /// Whether the names make an item: a later chunk on its own does not.
+    fn is_item(&self) -> bool {
+        self.chunks.contains_key(&0) || self.legacy.is_some()
+    }
+}
+
+#[cfg(test)]
+thread_local! {
+    /// Names [`group`] has looked at on this thread, so a test can bound the work of a listing.
+    static NAME_VISITS: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
+}
+
+/// The workbook's items whose lower-cased key starts with `prefix` (lower-cased), with the names
+/// that hold them, in one pass over the names.
+fn group<'a>(wb: &'a Workbook, prefix: &str) -> BTreeMap<String, Parts<'a>> {
+    let prefix = prefix.to_ascii_lowercase();
+    let mut items: BTreeMap<String, Parts<'a>> = BTreeMap::new();
+    for n in wb.names.iter() {
+        #[cfg(test)]
+        NAME_VISITS.with(|v| v.set(v.get().saturating_add(1)));
+        let Some((k, chunk)) = item_of(n) else { continue };
+        let k = k.to_ascii_lowercase();
+        if k.starts_with(&prefix) {
+            items.entry(k).or_default().add(chunk, &n.formula);
+        }
+    }
+    items
+}
 
 /// The item text stored under `key`, or `None` when there is none (or it is over the size limit).
 pub fn get(wb: &Workbook, key: &str) -> Option<String> {
-    let mut chunks: BTreeMap<u64, &str> = BTreeMap::new();
-    let mut legacy = None;
+    let mut parts = Parts::default();
     for n in wb.names.iter() {
-        match item_of(n) {
-            Some((k, Some(i))) if k.eq_ignore_ascii_case(key) => {
-                chunks.entry(i).or_insert(&n.formula);
-            }
-            Some((k, None)) if k.eq_ignore_ascii_case(key) => legacy = legacy.or(Some(&n.formula)),
-            _ => {}
+        if let Some((k, chunk)) = item_of(n)
+            && k.eq_ignore_ascii_case(key)
+        {
+            parts.add(chunk, &n.formula);
         }
     }
+    assemble(&parts)
+}
+
+/// What a listing found: the first [`MAX_ITEMS`] items or keys, and how many more there were.
+#[derive(Clone, Debug, Default, PartialEq, Eq)]
+pub struct Listing<T> {
+    pub found: T,
+    /// Items past the cap, not listed. A caller says so rather than letting them vanish.
+    pub more: usize,
+}
+
+/// The items whose key starts with `prefix` (`"macro_"`), by lower-cased key, read in one pass
+/// over the names: the first [`MAX_ITEMS`] items that can be read, and the count of the readable
+/// ones past them. An item over the size limit is neither listed nor counted, so it takes no
+/// place another item could have. ([`keys`] lists keys without reading the items.)
+pub fn items(wb: &Workbook, prefix: &str) -> Listing<BTreeMap<String, String>> {
+    let mut listing: Listing<BTreeMap<String, String>> = Listing::default();
+    for (k, parts) in group(wb, prefix) {
+        let Some(text) = parts.is_item().then(|| assemble(&parts)).flatten() else { continue };
+        if listing.found.len() < MAX_ITEMS {
+            listing.found.insert(k, text);
+        } else {
+            listing.more = listing.more.saturating_add(1);
+        }
+    }
+    listing
+}
+
+/// The text of the item held by `parts`, as [`get`] reads it.
+fn assemble(parts: &Parts) -> Option<String> {
+    let chunks = &parts.chunks;
     let mut out = String::new();
     if !chunks.contains_key(&0) {
-        out = string_literals(legacy?);
+        out = string_literals(parts.legacy?);
     } else {
         // Chunks run 0, 1, 2, …; anything after a gap isn't part of the item.
         for (want, (i, f)) in chunks.iter().enumerate() {
@@ -49,14 +140,47 @@ pub fn get(wb: &Workbook, key: &str) -> Option<String> {
                 break;
             }
             out.push_str(&string_literals(f));
-            if out.len() > MAX_ITEM_CHARS.saturating_mul(4) {
+            if out.len() > MAX_STORED_BYTES {
                 return None;
             }
         }
+        out = unescape(&out);
     }
     (out.chars().count() <= MAX_ITEM_CHARS).then_some(out)
 }
 
+/// Whether `c` is stored escaped: XML can't carry it, or it is the marker.
+fn needs_escape(c: char) -> bool {
+    c == ESCAPE || !matches!(c, '\t' | '\n' | '\r' | '\u{20}'..='\u{D7FF}' | '\u{E000}'..='\u{FFFD}' | '\u{10000}'..='\u{10FFFF}')
+}
+
+/// Turns [`ESCAPE`] and four hex digits back into the character. A marker not followed by four
+/// hex digits naming a character (a hand-made name) stays as it is.
+fn unescape(s: &str) -> String {
+    if !s.contains(ESCAPE) {
+        return s.to_string();
+    }
+    let mut out = String::with_capacity(s.len());
+    let mut rest = s;
+    while let Some(at) = rest.find(ESCAPE) {
+        out.push_str(rest.get(..at).unwrap_or(""));
+        let after = rest.get(at + ESCAPE.len_utf8()..).unwrap_or("");
+        let code = after.get(..4).filter(|h| h.bytes().all(|b| b.is_ascii_hexdigit())).and_then(|h| u32::from_str_radix(h, 16).ok());
+        match code.and_then(char::from_u32) {
+            Some(c) => {
+                out.push(c);
+                rest = after.get(4..).unwrap_or("");
+            }
+            None => {
+                out.push(ESCAPE);
+                rest = after;
+            }
+        }
+    }
+    out.push_str(rest);
+    out
+}
+
 /// Stores `text` under `key`, replacing what was there (in either format). Empty text removes it.
 pub fn set(wb: &mut Workbook, key: &str, text: &str) -> Result<()> {
     set_many(wb, &[(key.to_string(), text.to_string())])
@@ -76,7 +200,11 @@ pub fn set_many(wb: &mut Workbook, items: &[(String, String)]) -> Result<()> {
                 MAX_ITEM_CHARS / (1024 * 1024)
             )));
         }
-        keys.insert(key.to_ascii_lowercase());
+        // Two items under one key (names compare without case) would leave two hidden names of
+        // one name, of which an xlsx file keeps one: a caller's mistake, refused.
+        if !keys.insert(key.to_ascii_lowercase()) {
+            return Err(EngineError::Other(format!("`{key}` is given twice.")));
+        }
         for (i, formula) in chunk_formulas(text).into_iter().enumerate() {
             planned.push(DefinedName { name: format!("{PREFIX}{key}_{i}"), scope: None, formula, comment: "GridCraft data".into(), hidden: true });
         }
@@ -110,18 +238,21 @@ pub fn set_json(wb: &mut Workbook, key: &str, v: &Json) -> Result<()> {
     set(wb, key, &v.to_string())
 }
 
-/// The keys of the items stored in the workbook that start with `prefix` (`"macro_"`), sorted.
-pub fn keys(wb: &Workbook, prefix: &str) -> Vec<String> {
-    let mut keys: Vec<String> = wb
-        .names
-        .iter()
-        // A later chunk on its own is no item, as `get` reads it.
-        .filter_map(|n| item_of(n).filter(|(_, chunk)| chunk.is_none_or(|i| i == 0)).map(|(k, _)| k.to_ascii_lowercase()))
-        .filter(|k| k.starts_with(&prefix.to_ascii_lowercase()))
-        .collect();
-    keys.sort();
-    keys.dedup();
-    keys
+/// The keys of the items stored in the workbook that start with `prefix` (`"macro_"`), sorted
+/// and lower-cased: the first [`MAX_ITEMS`] of them. To read them all, [`items`] takes one pass
+/// where a [`get`] per key would take one each.
+pub fn keys(wb: &Workbook, prefix: &str) -> Listing<Vec<String>> {
+    let mut found: Vec<String> = group(wb, prefix).into_iter().filter(|(_, parts)| parts.is_item()).map(|(k, _)| k).collect();
+    let more = found.len().saturating_sub(MAX_ITEMS);
+    found.truncate(MAX_ITEMS);
+    Listing { found, more }
+}
+
+/// Whether the workbook holds an item under `name` (compared without case): a hidden workbook-scope
+/// `_gc_<key>_<k>` name, or a workbook-scope legacy `_sc_<key>` name, which is read whether or not
+/// it is hidden. A visible `_gc_` name is the person's own.
+pub fn holds_item(wb: &Workbook, name: &str) -> bool {
+    wb.names.iter().any(|n| n.name.eq_ignore_ascii_case(name) && item_of(n).is_some())
 }
 
 /// The item a hidden name belongs to, lower-cased: `_gc_<key>_<k>` and `_sc_<key>` give `<key>`.
@@ -165,7 +296,8 @@ fn check_key(key: &str) -> Result<()> {
 }
 
 /// `text` as name formulas: string literals of at most 255 characters joined with `&`, each formula
-/// at most 8,192 characters. Concatenating [`string_literals`] of each gives `text` back.
+/// at most 8,192 characters, with the characters XML can't carry escaped (see [`ESCAPE`]).
+/// Concatenating [`string_literals`] of each and turning the escapes back gives `text` back.
 pub fn chunk_formulas(text: &str) -> Vec<String> {
     let mut formulas = Vec::new();
     let (mut formula, mut units) = (String::new(), 0usize);
@@ -187,14 +319,22 @@ pub fn chunk_formulas(text: &str) -> Vec<String> {
         *lit_units = 0;
     };
     for ch in text.chars() {
-        if lit_units + ch.len_utf16() > LITERAL_UNITS {
+        // An escaped character is the marker and four digits, kept in one literal.
+        let escaped = needs_escape(ch);
+        let cost = if escaped { ESCAPE.len_utf16() + 4 } else { ch.len_utf16() };
+        if lit_units + cost > LITERAL_UNITS {
             flush_literal(&mut literal, &mut lit_units, &mut formula, &mut units);
         }
-        lit_units += ch.len_utf16();
-        if ch == '"' {
-            literal.push('"');
+        lit_units += cost;
+        if escaped {
+            literal.push(ESCAPE);
+            let _ = write!(literal, "{:04X}", u32::from(ch));
+        } else {
+            if ch == '"' {
+                literal.push('"');
+            }
+            literal.push(ch);
         }
-        literal.push(ch);
     }
     if lit_units > 0 {
         flush_literal(&mut literal, &mut lit_units, &mut formula, &mut units);
@@ -310,10 +450,10 @@ mod tests {
         // Only a later chunk: no item, not listed, and the legacy name is still read.
         wb.names.push(name("_gc_orphan_2", "\"c\""));
         assert_eq!(get(&wb, "orphan"), None);
-        assert_eq!(keys(&wb, "orphan"), Vec::<String>::new());
+        assert_eq!(keys(&wb, "orphan").found, Vec::<String>::new());
         wb.names.push(name("_sc_orphan", "\"old\""));
         assert_eq!(get(&wb, "orphan").as_deref(), Some("old"));
-        assert_eq!(keys(&wb, "orphan"), ["orphan"]);
+        assert_eq!(keys(&wb, "orphan").found, ["orphan"]);
         // Chunks out of order, a duplicate, junk around the literals, an unterminated literal.
         wb.names.push(name("_gc_mix_1", "\"b\"&A1&"));
         wb.names.push(name("_gc_mix_0", "SUM(\"a\")"));
@@ -361,7 +501,7 @@ mod tests {
         let own = DefinedName { hidden: false, ..name("_gc_watch_0", "\"mine\"") };
         wb.names.push(own.clone());
         assert_eq!(get(&wb, "watch"), None);
-        assert!(keys(&wb, "").is_empty());
+        assert!(keys(&wb, "").found.is_empty());
         // Writing the item would put a hidden name of the same name beside it, and an xlsx file
         // keeps only one of them: refused, with the names as they were.
         let err = set(&mut wb, "watch", "theirs").unwrap_err().to_string();
@@ -375,6 +515,18 @@ mod tests {
         assert_eq!(wb.names, vec![own]);
     }
 
+    #[test]
+    fn set_many_refuses_one_key_twice() {
+        // Two items under one key, in any case, would leave two hidden names of one name.
+        let mut wb = Workbook::new();
+        wb.names.push(name("_gc_other_0", "\"stays\""));
+        let err = set_many(&mut wb, &[("k".into(), "a".into()), ("K".into(), "b".into())]).unwrap_err().to_string();
+        assert!(err.contains("given twice"), "{err}");
+        assert_eq!(wb.names.len(), 1, "nothing changed");
+        set_many(&mut wb, &[("k".into(), "a".into()), ("k2".into(), "b".into())]).unwrap();
+        assert_eq!((get(&wb, "k").as_deref(), get(&wb, "K2").as_deref(), get(&wb, "other").as_deref()), (Some("a"), Some("b"), Some("stays")));
+    }
+
     #[test]
     fn a_chunk_number_with_leading_zeros_is_another_name() {
         // `_gc_watch_00` is not chunk 0 under another spelling: a hidden one is not read, and a
@@ -382,7 +534,7 @@ mod tests {
         let mut wb = Workbook::new();
         wb.names.push(name("_gc_watch_00", "\"theirs\""));
         assert_eq!(get(&wb, "watch"), None);
-        assert!(keys(&wb, "").is_empty());
+        assert!(keys(&wb, "").found.is_empty());
         let own = DefinedName { hidden: false, ..name("_gc_watch_00", "\"mine\"") };
         wb.names.push(own.clone());
         set(&mut wb, "watch", "ours").unwrap();
@@ -398,12 +550,74 @@ mod tests {
         set(&mut wb, "macro_2", "b").unwrap();
         set(&mut wb, "macro_1", &"a".repeat(9_000)).unwrap();
         set(&mut wb, "scripts", "{}").unwrap();
-        assert_eq!(keys(&wb, "macro_"), ["macro_1", "macro_2"]);
+        assert_eq!(keys(&wb, "macro_").found, ["macro_1", "macro_2"]);
         assert_eq!(item_key("_gc_macro_1_3").as_deref(), Some("macro_1"));
         assert_eq!(item_key("_SC_Scripts").as_deref(), Some("scripts"));
         assert_eq!(item_key("Total"), None);
     }
 
+    #[test]
+    fn listing_many_items_takes_one_pass_and_is_capped() {
+        // 20,000 hidden items, as a file can carry: a `get` per key looked at every name for each
+        // key (400 million visits); `items` looks at each name once, and lists at most MAX_ITEMS.
+        let mut wb = Workbook::new();
+        let total = 20_000;
+        for i in 0..total {
+            wb.names.push(name(&format!("_gc_macro_{i:05}_0"), &format!("\"m{i}\"")));
+        }
+        set(&mut wb, "macro_big", &"b".repeat(9_000)).unwrap();
+        wb.names.push(name("_sc_macro_legacy", "\"old\""));
+        wb.names.push(name("_gc_macro_orphan_1", "\"no chunk 0\""));
+        wb.names.push(name("_gc_other_0", "\"not a macro\""));
+        NAME_VISITS.with(|v| v.set(0));
+        let listing = items(&wb, "MACRO_");
+        let all = &listing.found;
+        assert_eq!(NAME_VISITS.with(|v| v.get()), wb.names.len(), "one pass over the names");
+        assert_eq!(all.len(), MAX_ITEMS);
+        let first: Vec<&String> = all.keys().take(2).collect();
+        assert_eq!(first, ["macro_00000", "macro_00001"]);
+        assert_eq!(all.get("macro_09999").map(String::as_str), Some("m9999"));
+        assert!(!all.contains_key("macro_10000"), "past the cap");
+        NAME_VISITS.with(|v| v.set(0));
+        let listed = keys(&wb, "macro_");
+        assert_eq!(NAME_VISITS.with(|v| v.get()), wb.names.len());
+        assert_eq!(listed.found, all.keys().cloned().collect::<Vec<_>>(), "keys and items agree");
+        assert_eq!((listing.more, listed.more), (10_002, 10_002), "the rest are counted, not lost");
+
+        // Under the cap every item is there, read as `get` reads it.
+        wb.names.retain(|n| !n.name.starts_with("_gc_macro_0") && !n.name.starts_with("_gc_macro_1"));
+        let few = items(&wb, "macro_").found;
+        let want: BTreeMap<String, String> =
+            [("macro_big", "b".repeat(9_000)), ("macro_legacy", "old".into())].into_iter().map(|(k, t)| (k.to_string(), t)).collect();
+        assert_eq!(few, want);
+        for (k, text) in &few {
+            assert_eq!(get(&wb, k).as_ref(), Some(text));
+        }
+        assert_eq!(keys(&wb, "macro_").found, ["macro_big", "macro_legacy"]);
+        assert_eq!((items(&wb, "").found.len(), items(&wb, "").more), (3, 0), "the other item too, none past the cap");
+    }
+
+    #[test]
+    fn an_item_over_the_size_limit_takes_no_place_in_a_listing() {
+        // One item too large to read sorts first and one sorts last, around MAX_ITEMS readable
+        // ones: every readable item is listed, and neither oversized one is counted as more.
+        let mut wb = Workbook::new();
+        let oversized = |wb: &mut Workbook, key: &str| {
+            for (i, formula) in chunk_formulas(&"x".repeat(MAX_ITEM_CHARS + 1)).into_iter().enumerate() {
+                wb.names.push(name(&format!("_gc_{key}_{i}"), &formula));
+            }
+        };
+        oversized(&mut wb, "macro_0");
+        for i in 0..MAX_ITEMS {
+            wb.names.push(name(&format!("_gc_macro_{i:05}_0"), &format!("\"m{i}\"")));
+        }
+        oversized(&mut wb, "macro_z");
+        assert_eq!(get(&wb, "macro_0"), None, "too large to read");
+        let listing = items(&wb, "macro_");
+        assert_eq!((listing.found.len(), listing.more), (MAX_ITEMS, 0));
+        assert!(listing.found.contains_key("macro_09999"));
+    }
+
     #[test]
     fn items_round_trip_through_xlsx() {
         let mut wb = Workbook::new();
@@ -415,5 +629,46 @@ mod tests {
         assert!(warnings.is_empty(), "{warnings:?}");
         assert_eq!(get(&back, "blob").as_deref(), Some(text.as_str()));
         assert_eq!(get_json(&back, "watch"), Some(serde_json::json!(["A1"])));
+
+        // Characters XML can't carry, and the escape marker itself, come back too, in text and
+        // in JSON (which leaves U+FFFE and U+FFFF unescaped).
+        let mut wb = Workbook::new();
+        let odd: String = (0u32..0x20).filter_map(char::from_u32).chain(['\u{FFFE}', '\u{FFFF}', ESCAPE, 'a', '\u{7F}', '\u{FDD1}']).collect();
+        let text: String = odd.chars().cycle().take(odd.chars().count() * 300).collect();
+        set(&mut wb, "odd", &text).unwrap();
+        excel_limits_hold(&wb);
+        let json = serde_json::json!({"s": "x\u{FFFF}y\u{1}z\u{FDD0}0041"});
+        set_json(&mut wb, "json", &json).unwrap();
+        // All escapes: each literal holds whole escapes, within the limits.
+        let controls = "\u{1}".repeat(9_000);
+        set(&mut wb, "controls", &controls).unwrap();
+        excel_limits_hold(&wb);
+        let bytes = gridcraft_xlsx::write_xlsx(&wb).unwrap();
+        let (back, warnings) = crate::io::open_bytes("data.xlsx", &bytes).unwrap();
+        assert!(warnings.is_empty(), "{warnings:?}");
+        assert_eq!(get(&back, "odd").as_deref(), Some(text.as_str()));
+        assert_eq!(get_json(&back, "json"), Some(json));
+        assert_eq!(get(&back, "controls").as_deref(), Some(controls.as_str()));
+    }
+
+    #[test]
+    fn escapes_count_against_the_limits_and_odd_markers_read_as_they_are() {
+        // Excel's limits hold however many characters are escaped, and the longest item of
+        // characters that all need escaping still reads.
+        for len in [1, 50, 51, 52, 1_638, 1_639, 5_000] {
+            let text = "\u{2}".repeat(len);
+            let fs = chunk_formulas(&text);
+            assert_eq!(unescape(&fs.iter().map(|f| string_literals(f)).collect::<String>()), text, "{len}");
+            assert!(fs.iter().all(|f| f.encode_utf16().count() <= FORMULA_UNITS), "{len}");
+        }
+        let mut wb = Workbook::new();
+        let longest = "\u{0}".repeat(MAX_ITEM_CHARS);
+        set(&mut wb, "longest", &longest).unwrap();
+        excel_limits_hold(&wb);
+        assert_eq!(get(&wb, "longest").map(|t| t.len()), Some(MAX_ITEM_CHARS));
+        // A marker in a hand-made name that is not an escape stays as it is.
+        let mut wb = Workbook::new();
+        wb.names.push(name("_gc_hand_0", "\"a\u{FDD0}zz\u{FDD0}D800\u{FDD0}004\u{FDD0}0041\u{FDD0}\""));
+        assert_eq!(get(&wb, "hand").as_deref(), Some("a\u{FDD0}zz\u{FDD0}D800\u{FDD0}004A\u{FDD0}"));
     }
 }

crates/engine/src/io.rs

@@ -166,7 +166,10 @@ fn read_plain(name: &str, bytes: &[u8], csv_as_text: bool) -> Result<(Workbook,
             let mut wb: Workbook = serde_json::from_slice(bytes).map_err(|e| EngineError::Other(format!("not a GridCraft JSON workbook: {e}")))?;
             wb.styles.rebuild_index();
             if wb.sheets.is_empty() {
+                // The file's macros are read after this (`macros::load`), so they come along.
+                let macros = std::mem::take(&mut wb.macros_read);
                 wb = Workbook::new();
+                wb.macros_read = macros;
             }
             // The file's band placement and cell count are derived state: rebuild them from the
             // cells themselves, dropping what is outside the sheet grid (#1008).
@@ -259,7 +262,7 @@ pub fn save_bytes_as(wb: &Workbook, path: &str, kind: FileKind) -> Result<Vec<u8
         // A workbook read from CSV keeps the separator it was read with (a `;` file stays `;`).
         FileKind::Csv => wb.sheet(sheet).map(|sh| csv(wb, sh, wb.csv_delimiter.unwrap_or(b','))).transpose().map(Option::unwrap_or_default),
         FileKind::Tsv => wb.sheet(sheet).map(|sh| csv(wb, sh, b'\t')).transpose().map(Option::unwrap_or_default),
-        FileKind::Json => serde_json::to_vec_pretty(wb).map_err(|e| EngineError::Other(e.to_string())),
+        FileKind::Json => crate::macros::json_bytes(wb),
         FileKind::Html => to_html(wb, sheet).map(String::into_bytes),
     }
 }

crates/engine/src/lib.rs

@@ -141,12 +141,13 @@ pub struct Clipboard {
     pub trust: Trust,
 }
 
-/// Whether a document's active content (macros) may run without asking the person first.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)]
+/// Whether a document's active content (macros) may run without asking the person first. There
+/// is no default: whoever makes a document says where it came from ([`DocState::new`]), so a new
+/// way of building one from bytes can't come out trusted by omission.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub enum Trust {
     /// Made in this session: a new workbook or a sample.
-    #[default]
     Local,
     /// Opened from a file, or holding content pasted from one: active content asks before it
     /// runs.
@@ -273,7 +274,10 @@ fn trim_to_budget(history: &mut Vec<HistoryEntry>, byte_budget: usize) {
 }
 
 impl DocState {
-    pub fn new(mut wb: Workbook, path: Option<String>, title: String) -> DocState {
+    /// A document for `wb`, which came from where `trust` says: [`Trust::FromFile`] for anything
+    /// read from bytes (a file, a download, a recovery copy), [`Trust::Local`] only for a workbook
+    /// made in this session.
+    pub fn new(mut wb: Workbook, path: Option<String>, title: String, trust: Trust) -> DocState {
         let mut calc = Calc::new();
         calc.recalc_all(&mut wb);
         let scripts_trusted = !cmd::extra::has_scripts(&wb);
@@ -294,7 +298,7 @@ impl DocState {
             unlocked_ranges: vec![],
             content_kind: None,
             scripts_trusted,
-            trust: Trust::Local,
+            trust,
             touched: None,
         }
     }
@@ -501,7 +505,7 @@ impl Session {
             let name = wb.next_sheet_name();
             wb.sheets.push(Arc::new(gridcraft_model::Sheet::new(name)));
         }
-        self.add_document(DocState::new(wb, None, format!("Book{n}")))
+        self.add_document(DocState::new(wb, None, format!("Book{n}"), Trust::Local))
     }
 
     /// Journals a command, then trims the oldest entries past either cap. A recording in progress
@@ -571,8 +575,15 @@ impl Session {
         let start = before.as_ref().and_then(|(uid, ..)| self.docs.iter().find(|d| d.uid == *uid).map(|d| (self.in_group(*uid), d.undo.len())));
         // A command that panics leaves the history as it found it: the steps commands nested in it
         // made or undid, and the redo history they cleared, come back. A length is not enough,
-        // because a nested `edit.undo` removes entries.
-        let history_before = before.as_ref().and_then(|(uid, ..)| self.docs.iter().find(|d| d.uid == *uid).map(|d| (d.undo.clone(), d.redo.clone())));
+        // because a nested `edit.undo` removes entries. A query runs nothing that changes the
+        // history (`every_query_leaves_the_history_alone` runs each one), so it needs no
+        // copy; front ends run queries all the time.
+        let query = !spec.journal && !spec.undoable;
+        let history_before = before.as_ref().filter(|_| !query).and_then(|(uid, ..)| {
+            #[cfg(test)]
+            tests::HISTORY_COPIES.with(|n| n.set(n.get().saturating_add(1)));
+            self.docs.iter().find(|d| d.uid == *uid).map(|d| (d.undo.clone(), d.redo.clone()))
+        });
         // The groups open on the document before the command (usually none), for the same reason.
         let groups_before: Vec<(u64, String, Arc<Workbook>, Selection)> = match &before {
             Some((uid, ..)) => {
@@ -586,6 +597,7 @@ impl Session {
             None => (0, 0, String::new(), Selection::default()),
         });
         let doc_uid = before.as_ref().map(|(uid, ..)| *uid);
+        let requests_before = self.ui_requests.len();
         let marked = doc_uid.zip(start).map(|(uid, (_, len))| (uid, len));
         self.marks.extend(marked);
         self.depth = self.depth.saturating_add(1);
@@ -631,10 +643,23 @@ impl Session {
                 return Err(EngineError::Internal(id.to_string(), msg));
             }
         };
+        // A command that fails leaves none of its own undo groups open (a `?` between
+        // `group_begin` and `group_end`): each is rolled back as `group_rollback` does. Only a
+        // command that succeeds keeps a group it opened (a debugger's start).
+        if result.is_err() {
+            self.rollback_groups_from(first_group);
+        }
         if spec.journal && result.is_ok() {
             // A command that started recording is not part of the recording; one that stopped it
-            // finds no recorder.
-            if let Some((doc, sheet, sheet_name, selection)) = place
+            // finds no recorder. A menu-style call that only opened a dialog (`file.open` with no
+            // file, `home.formatCells` with no style) is not recorded either: the dialog's own
+            // call is, so playing the recording back runs the command once, with its parameters.
+            // A call that opened a dialog and changed the workbook too (a script with an edit before
+            // a menu-style step) is recorded: the edit happened, and the dialog's confirming call
+            // is a step of its own.
+            let opened_dialog = self.ui_requests.get(requests_before..).is_some_and(|new| new.iter().any(|q| matches!(q, UiRequest::Dialog(..))));
+            let changed = before.as_ref().is_some_and(|(uid, wb, _)| self.docs.iter().any(|d| d.uid == *uid && !Arc::ptr_eq(&d.wb, wb)));
+            if let Some((doc, sheet, sheet_name, selection)) = place.filter(|_| (!opened_dialog || changed) && !record::NOT_RECORDED.contains(&id))
                 && let Some(rec) = self.recorder.as_mut()
             {
                 rec.push(record::RecordedStep { doc, sheet, sheet_name, selection, id: id.to_string(), params: cmd::journal_params(&params) });
@@ -788,6 +813,29 @@ impl Session {
         Ok(changed)
     }
 
+    /// Stops the running action on the document with uid `doc`: every undo group open on it is
+    /// rolled back as [`Session::group_rollback`] does, putting back the workbook, selection, undo
+    /// and redo history as they were when the outermost one began. Their tokens are then spent
+    /// (ending or rolling one back reports that it has already ended). Returns whether a group
+    /// was open. `automate.stopAction` runs it on the active document.
+    pub fn abort_groups(&mut self, doc: u64) -> bool {
+        let Some(id) = self.groups.iter().filter(|g| g.doc == doc).map(|g| g.id).min() else { return false };
+        // Rolling back the outermost group ends the ones opened inside it.
+        self.group_rollback(GroupToken { id, doc }).is_ok()
+    }
+
+    /// Rolls back every open undo group numbered `first` or later (the groups a failed command
+    /// opened), outermost first on each document.
+    fn rollback_groups_from(&mut self, first: u64) {
+        let opened: Vec<(u64, u64)> = self.groups.iter().filter(|g| g.id >= first).map(|g| (g.id, g.doc)).collect();
+        for (id, doc) in opened {
+            // A group inside one rolled back before it is already gone.
+            if self.groups.iter().any(|g| g.id == id) {
+                let _ = self.group_rollback(GroupToken { id, doc });
+            }
+        }
+    }
+
     /// Bounds the undo steps pushed while a command runs (a script of many steps, a subtotal per
     /// group): at most [`UNDO_LIMIT`] of them within [`UNDO_BYTES_LIMIT`], dropping the oldest.
     /// Only steps after the latest mark go: a running command's start, an open group's start and
@@ -858,6 +906,16 @@ impl Session {
         self.depth
     }
 
+    /// Runs a command as a command in progress would (one level deeper), for tests of what a
+    /// nested call may do.
+    #[cfg(test)]
+    pub(crate) fn execute_nested(&mut self, id: &str, params: Json) -> Result<Json> {
+        self.depth = self.depth.saturating_add(1);
+        let r = self.execute(id, params);
+        self.depth = self.depth.saturating_sub(1);
+        r
+    }
+
     /// Whether an undo group is open on the document with this uid.
     pub fn in_group(&self, doc: u64) -> bool {
         self.groups.iter().any(|g| g.doc == doc)
@@ -886,17 +944,12 @@ impl Session {
     /// queued a file dialog because it got no file (`file.open`, `file.saveAs`, `file.save` on a workbook with no
     /// file yet, `data.getData`) is an error instead of a silent success.
     pub fn execute_headless(&mut self, id: &str, params: Json) -> Result<Json> {
-        let recorded = self.recorder.as_ref().map(|r| (r.steps.len(), r.dropped));
         let r = self.execute(id, params);
         let requests = self.take_ui_requests();
         let asked_for_file = requests.iter().any(|q| matches!(q, UiRequest::Dialog(name, _) if name == "open" || name == "saveAs"));
+        // A call that only opened a dialog is not recorded (see `execute`), so the recording
+        // has no step to take back here.
         if r.is_ok() && asked_for_file {
-            // The caller gets an error, so the recording keeps no step for the command.
-            if let Some((steps, dropped)) = recorded
-                && let Some(rec) = self.recorder.as_mut()
-            {
-                rec.truncate(steps, dropped);
-            }
             let what = match id {
                 "file.open" => "`path` or `base64`",
                 "data.getData" | "data.fromTextCsv" => "`path` or `text`",

crates/engine/src/macros/mod.rs

@@ -52,7 +52,8 @@ pub fn check(m: &Macro) -> Result<(), String> {
 }
 
 /// Refuses a macro too large to keep in an xlsx file ([`stash::MAX_ITEM_CHARS`] characters of
-/// JSON), so saving a workbook never fails because of one of its macros.
+/// JSON), so a macro's size never makes a save fail. (A clashing visible name doesn't either:
+/// [`with_names`] stores the macro under another index.)
 fn check_size(m: &Macro) -> Result<(), String> {
     if macro_json(m).to_string().chars().count() > stash::MAX_ITEM_CHARS {
         return Err(format!("This macro is too large to keep in the workbook ({} characters at most).", stash::MAX_ITEM_CHARS));
@@ -105,34 +106,88 @@ pub fn fingerprint(m: &Macro) -> String {
 }
 
 /// A copy of `wb` for xlsx with each macro in hidden names (`_gc_macro_<n>_<k>`) instead of the
-/// field, or `None` when there is nothing to change.
+/// field, or `None` when there is nothing to change. The macros keep their order; the index `n`
+/// is this build's choice, so it steps around any a person's own visible name already uses.
 pub fn with_names(wb: &Workbook) -> crate::Result<Option<Workbook>> {
     // Only the numbered items are this build's: a hidden `_gc_macro_<x>_<k>` name with some
     // other key stays, as `load` leaves it.
     let stored = |n: &DefinedName| macro_item(n).is_some_and(|(_, i)| i.is_some());
-    if wb.macros.is_empty() && !wb.names.iter().any(stored) {
+    if wb.macros.is_empty() && wb.macros_unread.is_empty() && !wb.names.iter().any(stored) {
         return Ok(None);
     }
     let mut out = wb.clone();
     // One pass drops the items of the last save, and one writes the macros (a file can hold a
     // thousand macros and many thousands of names).
     out.names.retain(|n| !stored(n));
-    let items: Vec<(String, String)> = wb.macros.iter().enumerate().map(|(i, m)| (format!("{MACRO_KEY}{i}"), macro_json(m).to_string())).collect();
+    // A visible workbook-scope name of the storage shape is someone's own, and `set_many` won't
+    // write a hidden one of the same name beside it. Such a name (`_GC_MACRO_1_0`, or one a
+    // person defined after the file opened) blocks its item's key, in any chunk, so the macros
+    // take the free indices from 0 up. The keys compare as the stash reads them, so this skips
+    // exactly what `set_many` would refuse, and no arithmetic depends on a number from the file.
+    let blocked: HashSet<String> = out.names.iter().filter(|n| !n.hidden && n.scope.is_none()).filter_map(|n| stash::item_key(&n.name)).collect();
+    let free = (0..=u64::MAX).map(|i| format!("{MACRO_KEY}{i}")).filter(|key| !blocked.contains(key));
+    // Macros in a newer format follow the readable ones, written back as the file held them.
+    let texts = wb.macros.iter().map(|m| macro_json(m).to_string()).chain(wb.macros_unread.iter().map(Json::to_string));
+    let items: Vec<(String, String)> = texts.zip(free).map(|(text, key)| (key, text)).collect();
     stash::set_many(&mut out, &items)?;
-    out.macros.clear();
+    // An empty list of its own, not a copy of the shared one cleared.
+    out.macros = gridcraft_model::Macros::default();
+    out.macros_unread = std::sync::Arc::default();
     Ok(Some(out))
 }
 
-/// After a file is opened: moves macros out of hidden names into `Workbook::macros` and drops
-/// macros that are damaged, over the limits or in a newer format. Returns a warning for what was dropped.
+/// The JSON format's bytes for `wb`: the workbook, with the macros in a newer format put back into
+/// the `macros` list as the file held them.
+pub fn json_bytes(wb: &Workbook) -> crate::Result<Vec<u8>> {
+    // Every path that adds a macro checks it, so this refuses only one that skipped the check, as
+    // the xlsx save does, instead of writing a macro the next open would drop.
+    for m in &wb.macros {
+        check(m).map_err(crate::EngineError::Other)?;
+    }
+    let bytes = if wb.macros_unread.is_empty() {
+        serde_json::to_vec_pretty(wb)
+    } else {
+        serde_json::to_value(wb).and_then(|mut v| {
+            if let Some(list) = v.as_object_mut().map(|o| o.entry("macros").or_insert_with(|| Json::Array(vec![]))).and_then(Json::as_array_mut) {
+                list.extend(wb.macros_unread.iter().cloned());
+            }
+            serde_json::to_vec_pretty(&v)
+        })
+    };
+    bytes.map_err(|e| crate::EngineError::Other(e.to_string()))
+}
+
+/// Whether `v` is a macro in a newer format than this build reads.
+fn is_newer(v: &Json) -> bool {
+    v.get("v").and_then(Json::as_u64).is_some_and(|found| found > u64::from(gridcraft_model::MACRO_VERSION))
+}
+
+/// Keeps a macro in a newer format unread, if it is small enough to save back
+/// ([`stash::MAX_ITEM_CHARS`] characters of JSON, the size every xlsx save can write).
+fn keep_unread(unread: &mut Vec<Json>, v: Json, bad: &mut usize) {
+    if v.to_string().chars().count() <= stash::MAX_ITEM_CHARS {
+        unread.push(v);
+    } else {
+        *bad = bad.saturating_add(1);
+    }
+}
+
+/// After a file is opened: moves macros out of hidden names into `Workbook::macros`, keeps
+/// macros in a newer format unread (`Workbook::macros_unread`), and drops macros that are damaged
+/// or over the limits. Returns a warning for what was kept unread and one for what was dropped.
 pub fn load(wb: &mut Workbook) -> Vec<String> {
     let mut warnings = Vec::new();
     let mut bad = 0usize;
+    let mut unread = Vec::new();
     // Entries of the JSON format's field, each read on its own so a damaged one drops alone.
     let read = std::mem::take(&mut wb.macros_read);
     bad = bad.saturating_add(read.len().saturating_sub(gridcraft_model::MAX_MACROS));
-    for v in read.iter().take(gridcraft_model::MAX_MACROS) {
-        match parse_macro(v) {
+    for v in read.into_iter().take(gridcraft_model::MAX_MACROS) {
+        if is_newer(&v) {
+            keep_unread(&mut unread, v, &mut bad);
+            continue;
+        }
+        match parse_macro(&v) {
             Ok(m) => wb.macros.push(m),
             Err(_) => bad += 1,
         }
@@ -156,7 +211,12 @@ pub fn load(wb: &mut Workbook) -> Vec<String> {
     let mut item = Workbook::new();
     for ((_, key), names) in stored.into_iter().take(gridcraft_model::MAX_MACROS) {
         item.names = names;
-        match stash::get_json(&item, &key).ok_or_else(|| "unreadable".to_string()).and_then(|v| parse_macro(&v)) {
+        let v = stash::get_json(&item, &key);
+        if let Some(v) = v.as_ref().filter(|v| is_newer(v)) {
+            keep_unread(&mut unread, v.clone(), &mut bad);
+            continue;
+        }
+        match v.ok_or_else(|| "unreadable".to_string()).and_then(|v| parse_macro(&v)) {
             Ok(m) => found.push(m),
             Err(_) => bad += 1,
         }
@@ -184,9 +244,25 @@ pub fn load(wb: &mut Workbook) -> Vec<String> {
             Err(_) => bad += 1,
         }
     }
+    // Readable and unread macros together stay within the limit, the readable ones first.
+    let room = gridcraft_model::MAX_MACROS.saturating_sub(wb.macros.len());
+    bad = bad.saturating_add(unread.len().saturating_sub(room));
+    unread.truncate(room);
+    if !unread.is_empty() {
+        let one = unread.len() == 1;
+        warnings.push(format!(
+            "{} macro{} in the workbook {} made by a newer version of GridCraft: this version can't show or run {}, and saving keeps {} unchanged.",
+            unread.len(),
+            if one { "" } else { "s" },
+            if one { "was" } else { "were" },
+            if one { "it" } else { "them" },
+            if one { "it" } else { "them" },
+        ));
+    }
+    wb.macros_unread = std::sync::Arc::new(unread);
     if bad > 0 {
         warnings.push(format!(
-            "{bad} macro{} in the workbook couldn't be opened: {} damaged, made by a newer version of GridCraft, or over the limits ({} macros, {} steps each, Repeat and If nested {} deep).",
+            "{bad} macro{} in the workbook couldn't be opened: {} damaged or over the limits ({} macros, {} steps each, Repeat and If nested {} deep).",
             if bad == 1 { "" } else { "s" },
             if bad == 1 { "it is" } else { "they are" },
             gridcraft_model::MAX_MACROS,

crates/engine/src/macros/resolve.rs

@@ -288,9 +288,12 @@ pub fn resolve_token(text: &str, env: &Env) -> R<Place> {
                 return bad(format!("'{t}': '{name}' with these arguments isn't a token method ({}).", METHODS.join(", ")));
             }
         };
-        // After a method the result is plain cells: `data` no longer applies, and only `data`
-        // keeps the header row for `col("…")`.
-        header = false;
+        // After a method the result is plain cells, except that `col` and `cols` keep the input's top
+        // row, so a later `data` still takes the header off; only `data` keeps the header row for
+        // `col("…")`.
+        if !matches!(method.as_str(), "col" | "cols") {
+            header = false;
+        }
         if method != "data" {
             names_row = None;
         }
@@ -615,7 +618,9 @@ mod tests {
             ("input", "B2:D11"),
             ("Input.col(2)", "C2:C11"),
             ("Input.col(\"Amount\")", "C2:C11"),
-            ("Input.col(\"date\").data", "D2:D11"),
+            ("Input.col(\"date\").data", "D3:D11"),
+            ("Input.cols(2..3).data", "C3:D11"),
+            ("Input.col(2).data.col(1)", "C3:C11"),
             ("Input.data", "B3:D11"),
             ("Input.data.col(2)", "C3:C11"),
             ("Input.data.col(\"Amount\")", "C3:C11"),

crates/engine/src/macros/tests.rs

@@ -213,24 +213,82 @@ fn a_macro_built_in_code_goes_through_the_size_check_too() {
     assert!(check(&big).unwrap_err().contains("too large"));
     let mut wb = Workbook::new();
     wb.macros.push(big);
-    assert!(crate::io::save_bytes(&wb, "book.xlsx").is_err(), "a macro that fails `check` fails the save");
+    // A macro that fails `check` fails the save in either format, rather than being written and
+    // dropped the next time the file opens.
+    for file in ["book.xlsx", "book.json"] {
+        assert!(crate::io::save_bytes(&wb, file).is_err(), "{file}");
+    }
+}
+
+fn visible(name: &str) -> DefinedName {
+    DefinedName { name: name.into(), scope: None, formula: "Sheet1!$A$1".into(), comment: String::new(), hidden: false }
+}
+
+/// Saves `wb` to xlsx and returns the names in the file and the workbook opened from it.
+fn xlsx_round_trip(wb: &Workbook) -> (Vec<String>, Workbook) {
+    let bytes = crate::io::save_bytes(wb, "book.xlsx").unwrap();
+    let raw = names_of(&gridcraft_xlsx::read_xlsx(&bytes).unwrap().0);
+    let (back, warnings) = crate::io::open_bytes("book.xlsx", &bytes).unwrap();
+    assert!(warnings.is_empty(), "{warnings:?}");
+    (raw, back)
 }
 
 #[test]
-fn a_save_refuses_a_clashing_visible_name_before_changing_anything() {
+fn a_visible_name_of_the_storage_shape_does_not_block_a_save() {
+    // The index a macro is stored under is GridCraft's choice, so a person's own visible name
+    // of that shape (here in capitals: names compare without case) is stepped around, not
+    // refused. Stale hidden items still go in the same pass.
     let mut wb = Workbook::new();
     wb.macros.push(sample_macro("A"));
     wb.macros.push(sample_macro("B"));
-    let own = DefinedName { hidden: false, ..hidden("_GC_MACRO_1_0", "mine") };
-    wb.names.push(own.clone());
+    wb.names.push(visible("_GC_MACRO_0_0"));
     wb.names.push(hidden("_gc_macro_7_0", "stale"));
-    let err = crate::io::save_bytes(&wb, "book.xlsx").unwrap_err().to_string();
-    assert!(err.contains("_GC_MACRO_1_0"), "{err}");
-    assert_eq!(wb.names.len(), 2, "nothing changed");
-    wb.names.remove(0);
-    let (raw, _) = gridcraft_xlsx::read_xlsx(&crate::io::save_bytes(&wb, "book.xlsx").unwrap()).unwrap();
-    let names = names_of(&raw);
-    assert_eq!(names, ["_gc_macro_0_0", "_gc_macro_1_0"], "the stale item went in the same pass");
+    let (raw, back) = xlsx_round_trip(&wb);
+    assert_eq!(raw, ["_GC_MACRO_0_0", "_gc_macro_1_0", "_gc_macro_2_0"]);
+    assert_eq!(back.macros, wb.macros, "both macros, in their order");
+    assert_eq!(back.names, vec![visible("_GC_MACRO_0_0")], "the person's name is kept as it was");
+    assert_eq!(wb.names.len(), 2, "the open workbook is not changed by a save");
+}
+
+#[test]
+fn a_huge_visible_index_takes_no_part_in_numbering() {
+    // A file can carry a visible name with any number in it. The macros take the free indices
+    // from 0 up, so no arithmetic depends on that number (one past the largest would overflow).
+    let mut wb = Workbook::new();
+    wb.macros.push(sample_macro("A"));
+    wb.names.push(visible(&format!("_gc_macro_{}_0", u64::MAX)));
+    wb.names.push(visible("_gc_macro_0_0"));
+    let (raw, back) = xlsx_round_trip(&wb);
+    assert_eq!(raw, [format!("_gc_macro_{}_0", u64::MAX), "_gc_macro_0_0".into(), "_gc_macro_1_0".into()]);
+    assert_eq!(back.macros, wb.macros);
+    assert_eq!(names_of(&back), names_of(&wb));
+}
+
+#[test]
+fn a_visible_legacy_name_of_the_storage_shape_is_kept() {
+    // The stash reads a legacy `_sc_<key>` name as data whether or not it is hidden, so writing
+    // key `macro_0` would remove a person's visible `_sc_macro_0`. That key is stepped around too.
+    let mut wb = Workbook::new();
+    wb.macros.push(sample_macro("A"));
+    wb.names.push(visible("_sc_macro_0"));
+    let (raw, back) = xlsx_round_trip(&wb);
+    assert_eq!(raw, ["_sc_macro_0", "_gc_macro_1_0"]);
+    assert_eq!(back.macros, wb.macros);
+    assert_eq!(back.names, vec![visible("_sc_macro_0")]);
+}
+
+#[test]
+fn a_name_defined_after_opening_does_not_block_the_next_save() {
+    // Once a file is open its macros are out of the names, so Define Name accepts a visible
+    // `_gc_macro_0_0`; the next save steps around it.
+    let mut s = crate::Session::new();
+    s.execute("file.new", json!({})).unwrap();
+    std::sync::Arc::make_mut(&mut s.doc_mut().unwrap().wb).macros.push(sample_macro("A"));
+    s.execute("formulas.defineName", json!({"name": "_gc_macro_0_0", "refersTo": "=Sheet1!$A$1"})).unwrap();
+    let (raw, back) = xlsx_round_trip(&s.doc().unwrap().wb);
+    assert_eq!(raw, ["_gc_macro_0_0", "_gc_macro_1_0"]);
+    assert_eq!(back.macros.len(), 1);
+    assert_eq!(names_of(&back), ["_gc_macro_0_0"]);
 }
 
 #[test]
@@ -272,7 +330,8 @@ fn damaged_or_oversized_macros_are_dropped_with_a_warning() {
     let mut wb = Workbook::new();
     wb.macros = (0..MAX_MACROS + 5).map(|i| Macro { name: format!("M{i}"), ..Macro::default() }).collect();
     wb.macros[0].steps = vec![Step::command("x", json!({})); MAX_MACRO_STEPS + 1];
-    let bytes = crate::io::save_bytes(&wb, "many.json").unwrap();
+    // Written as another program would: a GridCraft save refuses a macro that fails `check`.
+    let bytes = serde_json::to_vec(&wb).unwrap();
     let (back, warnings) = crate::io::open_bytes("many.json", &bytes).unwrap();
     // Only the first MAX_MACROS entries are read, as in xlsx.
     assert_eq!(back.macros.len(), MAX_MACROS - 1);
@@ -285,7 +344,8 @@ fn damaged_or_oversized_macros_are_dropped_with_a_warning() {
     assert!(err.contains("too large"), "{err}");
     let mut wb = Workbook::new();
     wb.macros.push(Macro { name: "Huge".into(), steps: vec![Step::command("cell.set", json!({"input": huge}))], ..Macro::default() });
-    let bytes = crate::io::save_bytes(&wb, "huge.json").unwrap();
+    // Written as another program would: a GridCraft save refuses a macro that fails `check`.
+    let bytes = serde_json::to_vec(&wb).unwrap();
     let (back, warnings) = crate::io::open_bytes("huge.json", &bytes).unwrap();
     assert!(back.macros.is_empty() && warnings[0].starts_with("1 macro "), "{warnings:?}");
 }
@@ -319,7 +379,8 @@ fn json_field_macros_get_the_name_check_too() {
     let mut wb = Workbook::new();
     wb.macros.push(Macro { name: "a\nb".into(), ..Macro::default() });
     wb.macros.push(Macro { name: "Fine".into(), ..Macro::default() });
-    let bytes = crate::io::save_bytes(&wb, "ctl.json").unwrap();
+    // Written as another program would: a GridCraft save refuses a macro that fails `check`.
+    let bytes = serde_json::to_vec(&wb).unwrap();
     let (back, warnings) = crate::io::open_bytes("ctl.json", &bytes).unwrap();
     let names: Vec<&str> = back.macros.iter().map(|m| m.name.as_str()).collect();
     assert_eq!(names, ["Fine"]);
@@ -391,10 +452,11 @@ fn a_damaged_json_macro_drops_alone() {
 }
 
 #[test]
-fn a_macro_in_a_newer_format_is_dropped_not_rewritten() {
+fn a_macro_in_a_newer_format_is_kept_unread_and_saved_back_unchanged() {
     // A newer build's macro (`v` 2, with a field this build doesn't know) would lose that field
-    // here and be saved back as if complete, so it is dropped with the usual warning instead.
-    // Version 1 and a missing version are read as before.
+    // if it were read here, so it is never read: it is kept as the file held it, saved back
+    // unchanged, and the newer build gets it back whole. Version 1 and a missing version are
+    // read as before.
     let future = json!({"name": "Future", "v": MACRO_VERSION + 1, "fromALaterBuild": {"x": 1}, "steps": [{"command": "cell.set"}]});
     let current = json!({"name": "Current", "v": MACRO_VERSION, "steps": [{"command": "cell.set"}]});
     let missing = json!({"name": "Missing"});
@@ -407,17 +469,24 @@ fn a_macro_in_a_newer_format_is_dropped_not_rewritten() {
         let names: Vec<&str> = back.macros.iter().map(|m| m.name.as_str()).collect();
         assert_eq!(names, ["Current", "Missing"], "{file}");
         assert!(back.macros.iter().all(|m| m.v == MACRO_VERSION), "{file}");
-        assert!(warnings.len() == 1 && warnings[0].starts_with("1 macro ") && warnings[0].contains("newer version"), "{file}: {warnings:?}");
-        // Saving and opening again leaves no reduced copy of it behind.
+        assert_eq!(*back.macros_unread, std::slice::from_ref(&future), "{file}");
+        assert!(
+            warnings.len() == 1 && warnings[0].starts_with("1 macro ") && warnings[0].contains("newer version") && warnings[0].contains("unchanged"),
+            "{file}: {warnings:?}"
+        );
+        // Saved and opened again, it is still there, whole, and the others are as they were.
         let (again, warnings) = crate::io::open_bytes(file, &crate::io::save_bytes(back, file).unwrap()).unwrap();
-        assert!(warnings.is_empty(), "{file}: {warnings:?}");
+        assert_eq!(warnings.len(), 1, "{file}: {warnings:?}");
         assert_eq!(again.macros, back.macros, "{file}");
+        assert_eq!(again.macros_unread, back.macros_unread, "{file}");
     };
     // The JSON format's field.
     let mut v: serde_json::Value = serde_json::from_slice(&crate::io::save_bytes(&Workbook::new(), "book.json").unwrap()).unwrap();
     v["macros"] = json!([future, current, missing]);
     let (back, warnings) = crate::io::open_bytes("book.json", v.to_string().as_bytes()).unwrap();
     check("book.json", &back, &warnings);
+    let saved: serde_json::Value = serde_json::from_slice(&crate::io::save_bytes(&back, "book.json").unwrap()).unwrap();
+    assert!(saved["macros"].as_array().unwrap().contains(&future), "the file holds it as it was");
     // Hidden names in xlsx.
     let mut wb = Workbook::new();
     for (i, m) in [&future, &current, &missing].into_iter().enumerate() {
@@ -429,6 +498,42 @@ fn a_macro_in_a_newer_format_is_dropped_not_rewritten() {
     check("book.xlsx", &back, &warnings);
 }
 
+#[test]
+fn kept_macros_share_the_save_limits_and_the_numbering() {
+    // A newer macro too large for the file is left out (keeping it would make every xlsx save
+    // fail), and readable and unread macros together stay within the macro limit, readable ones
+    // first. Kept ones are numbered after the readable ones, around a person's visible name.
+    let newer = |name: &str| json!({"name": name, "v": MACRO_VERSION + 1});
+    let mut big = newer("Big");
+    big["pad"] = json!(long(crate::cmd::stash::MAX_ITEM_CHARS));
+    let mut file: serde_json::Value = serde_json::from_slice(&crate::io::save_bytes(&Workbook::new(), "book.json").unwrap()).unwrap();
+    let mut list = vec![big, newer("Kept"), json!({"name": "Read"})];
+    file["macros"] = json!(list);
+    let (mut wb, warnings) = crate::io::open_bytes("book.json", file.to_string().as_bytes()).unwrap();
+    assert_eq!(*wb.macros_unread, [newer("Kept")]);
+    assert_eq!(warnings.len(), 2, "one kept, one left out: {warnings:?}");
+    wb.names.push(visible("_gc_macro_1_0"));
+    let (raw, back) = {
+        let bytes = crate::io::save_bytes(&wb, "book.xlsx").unwrap();
+        let raw = names_of(&gridcraft_xlsx::read_xlsx(&bytes).unwrap().0);
+        (raw, crate::io::open_bytes("book.xlsx", &bytes).unwrap().0)
+    };
+    assert_eq!(raw, ["_gc_macro_1_0", "_gc_macro_0_0", "_gc_macro_2_0"]);
+    assert_eq!((back.macros.len(), back.macros_unread.len()), (1, 1));
+    // Past the limit the rest are left out; and when a JSON file's list is full, an unread item
+    // it also holds in hidden names is the one left out: readable macros come first.
+    list = (0..MAX_MACROS).map(|i| json!({"name": format!("M{i}")})).collect();
+    let mut with_item = Workbook::new();
+    with_item.names.push(hidden("_gc_macro_0_0", &newer("InNames").to_string()));
+    let mut file: serde_json::Value = serde_json::from_slice(&crate::io::save_bytes(&with_item, "book.json").unwrap()).unwrap();
+    for extra in [None, Some(newer("Over"))] {
+        file["macros"] = json!(list.iter().cloned().chain(extra).collect::<Vec<_>>());
+        let (wb, warnings) = crate::io::open_bytes("book.json", file.to_string().as_bytes()).unwrap();
+        assert_eq!((wb.macros.len(), wb.macros_unread.len()), (MAX_MACROS, 0));
+        assert!(warnings.len() == 1 && warnings[0].contains("couldn't be opened"), "{warnings:?}");
+    }
+}
+
 #[test]
 fn many_macros_with_one_name_all_survive_save_and_reopen() {
     // MAX_MACROS macros with the same name, at the length limit, so the suffixes run from " (2)"
@@ -478,3 +583,149 @@ fn naming_clashing_macros_takes_linear_work() {
     taken.insert("a (2)");
     assert_eq!(taken.unique("a"), "a (4)");
 }
+
+#[test]
+fn a_json_workbook_with_no_sheets_keeps_its_macros() {
+    // A JSON file with an empty sheet list opens as a new workbook; its macros come along.
+    let mut wb = Workbook::new();
+    wb.macros.push(sample_macro("A"));
+    let mut v: serde_json::Value = serde_json::from_slice(&crate::io::save_bytes(&wb, "book.json").unwrap()).unwrap();
+    v["sheets"] = json!([]);
+    let (back, warnings) = crate::io::open_bytes("book.json", v.to_string().as_bytes()).unwrap();
+    assert!(warnings.is_empty(), "{warnings:?}");
+    assert_eq!(back.sheets.len(), 1);
+    assert_eq!(back.macros, wb.macros);
+}
+
+fn long(n: usize) -> String {
+    "x".repeat(n)
+}
+
+/// `sample_macro` as JSON with `change` made to it.
+fn changed(change: fn(&mut serde_json::Value)) -> serde_json::Value {
+    let mut v = macro_json(&sample_macro("T"));
+    change(&mut v);
+    v
+}
+
+#[test]
+fn every_field_a_file_sets_has_a_limit() {
+    // The sample's steps: 0 a command, 1 a Repeat whose body holds an If whose else holds a Stop.
+    let refused: [(&str, fn(&mut serde_json::Value)); 14] = [
+        ("shortcut", |v| v["shortcut"] = json!(long(MAX_MACRO_SHORTCUT + 1))),
+        ("shortcut line break", |v| v["shortcut"] = json!("Ctrl+\nM")),
+        ("permission count", |v| v["permissions"] = json!(vec!["data.getData"; MAX_MACRO_PERMISSIONS + 1])),
+        ("permission text", |v| v["permissions"] = json!(["data getData"])),
+        ("output anchor", |v| v["output"]["anchor"] = json!(long(MAX_MACRO_REF + 1))),
+        ("default range", |v| v["inputs"][0]["defaultRef"] = json!(long(MAX_MACRO_REF + 1))),
+        ("column types", |v| {
+            v["inputs"][0]["cols"] = json!([1, 0]);
+            v["inputs"][0]["colTypes"] = json!(vec!["any"; MAX_MACRO_COLUMNS + 1]);
+        }),
+        ("command id", |v| v["steps"][0]["id"] = json!("cell set")),
+        ("empty command id", |v| v["steps"][0]["id"] = json!("")),
+        ("Repeat range", |v| v["steps"][1]["over"] = json!(long(MAX_MACRO_REF + 1))),
+        ("If cell", |v| v["steps"][1]["body"][0]["cell"] = json!(long(MAX_MACRO_REF + 1))),
+        // As JSON the string takes its two quotes as well.
+        ("If value", |v| v["steps"][1]["body"][0]["value"] = json!(long(MAX_MACRO_TEXT - 1))),
+        ("Stop message", |v| v["steps"][1]["body"][0]["else"][0]["message"] = json!(long(MAX_MACRO_TEXT + 1))),
+        ("Stop message, nested", |v| v["steps"][1]["body"][0]["else"][0]["message"] = json!(long(MAX_MACRO_TEXT * 2))),
+    ];
+    for (what, change) in refused {
+        assert!(parse_macro(&changed(change)).is_err(), "{what} past its limit is refused");
+    }
+    let accepted: [(&str, fn(&mut serde_json::Value)); 9] = [
+        ("shortcut", |v| v["shortcut"] = json!(long(MAX_MACRO_SHORTCUT))),
+        ("permission count", |v| v["permissions"] = json!(vec!["data.getData"; MAX_MACRO_PERMISSIONS])),
+        ("output anchor", |v| v["output"]["anchor"] = json!(long(MAX_MACRO_REF))),
+        ("column types", |v| {
+            v["inputs"][0]["cols"] = json!([1, 0]);
+            v["inputs"][0]["colTypes"] = json!(vec!["any"; MAX_MACRO_COLUMNS]);
+        }),
+        ("If value", |v| v["steps"][1]["body"][0]["value"] = json!(long(MAX_MACRO_TEXT - 2))),
+        // Empty and NotEmpty ignore the value, which is then left out (null).
+        ("If with no value", |v| {
+            v["steps"][1]["body"][0]["op"] = json!("empty");
+            v["steps"][1]["body"][0].as_object_mut().unwrap().remove("value");
+        }),
+        // How many times a Repeat may run is the runner's to bound, not the format's.
+        ("Repeat max", |v| v["steps"][1]["max"] = json!(u32::MAX)),
+        ("Stop message", |v| v["steps"][1]["body"][0]["else"][0]["message"] = json!(long(MAX_MACRO_TEXT))),
+        ("no shortcut", |v| {
+            v.as_object_mut().unwrap().remove("shortcut");
+        }),
+    ];
+    for (what, change) in accepted {
+        assert!(parse_macro(&changed(change)).is_ok(), "{what} at its limit is accepted");
+    }
+    // Every command there is passes as a step and as a permission.
+    for spec in crate::command_specs() {
+        let m = json!({"name": "T", "steps": [{"command": spec.id}], "permissions": [spec.id]});
+        assert!(parse_macro(&m).is_ok(), "{}", spec.id);
+    }
+}
+
+#[test]
+fn an_input_declaration_is_checked_as_a_whole() {
+    let input = |i: serde_json::Value| json!({"name": "T", "inputs": [i]});
+    for i in [
+        json!({"name": "Input", "rows": [5, 2]}),
+        json!({"name": "Input", "cols": [3, 2]}),
+        json!({"name": "Input", "kind": "cell", "cols": [2, 5]}),
+        json!({"name": "Input", "kind": "cell", "rows": [0, 0]}),
+        json!({"name": "Input", "kind": "cell", "header": true}),
+        json!({"name": "Input", "kind": "cell", "colTypes": ["number", "text"]}),
+        json!({"name": "Input", "cols": [2, 2], "colTypes": ["number", "text", "any"]}),
+    ] {
+        assert!(parse_macro(&input(i.clone())).is_err(), "{i}");
+    }
+    for i in [
+        json!({"name": "Input", "rows": [5, 0]}),
+        json!({"name": "Input", "rows": [2, 2], "cols": [0, 3]}),
+        json!({"name": "Input", "kind": "cell"}),
+        json!({"name": "Input", "kind": "cell", "rows": [1, 1], "cols": [1, 1], "colTypes": ["number"]}),
+        json!({"name": "Input", "cols": [1, 2], "colTypes": ["number", "text"]}),
+    ] {
+        assert!(parse_macro(&input(i.clone())).is_ok(), "{i}");
+    }
+    // From a file, a macro whose declaration fails is left out with the usual warning, and the
+    // workbook's other macros open.
+    let mut file: serde_json::Value = serde_json::from_slice(&crate::io::save_bytes(&Workbook::new(), "book.json").unwrap()).unwrap();
+    file["macros"] = json!([input(json!({"name": "Input", "rows": [5, 2]})), {"name": "Fine"}]);
+    let (wb, warnings) = crate::io::open_bytes("book.json", file.to_string().as_bytes()).unwrap();
+    assert_eq!(wb.macros.iter().map(|m| m.name.as_str()).collect::<Vec<_>>(), ["Fine"]);
+    assert_eq!(warnings.len(), 1, "{warnings:?}");
+}
+
+#[test]
+fn the_storage_rules_hold_together_through_both_formats() {
+    // A visible name of the storage shape, a macro in a newer format and one whose input breaks
+    // the declaration rules, in one file, saved and opened again in each format and then the
+    // other: the visible name stays, the newer macro comes back whole, the broken one is left out
+    // once, and the readable one survives.
+    let newer = json!({"name": "Kept", "v": MACRO_VERSION + 1, "fromALaterBuild": true});
+    let broken = json!({"name": "Broken", "inputs": [{"name": "Input", "rows": [5, 2]}]});
+    let mut start = Workbook::new();
+    start.names.push(visible("_gc_macro_0_0"));
+    let mut file: serde_json::Value = serde_json::from_slice(&crate::io::save_bytes(&start, "book.json").unwrap()).unwrap();
+    file["macros"] = json!([newer, broken, {"name": "Good"}]);
+    let (wb, warnings) = crate::io::open_bytes("book.json", file.to_string().as_bytes()).unwrap();
+    assert_eq!(warnings.len(), 2, "one kept, one left out: {warnings:?}");
+    let expect = |wb: &Workbook, how: &str| {
+        assert_eq!(wb.macros.iter().map(|m| m.name.as_str()).collect::<Vec<_>>(), ["Good"], "{how}");
+        assert_eq!(*wb.macros_unread, std::slice::from_ref(&newer), "{how}");
+        assert_eq!(names_of(wb), ["_gc_macro_0_0"], "{how}");
+    };
+    expect(&wb, "opened");
+    for formats in [["book.xlsx", "book.json"], ["book.json", "book.xlsx"]] {
+        let mut now = wb.clone();
+        for file in formats {
+            let (back, warnings) = crate::io::open_bytes(file, &crate::io::save_bytes(&now, file).unwrap()).unwrap();
+            assert_eq!(warnings.len(), 1, "{file}: only the kept macro is reported again: {warnings:?}");
+            expect(&back, file);
+            now = back;
+        }
+    }
+    let bytes = crate::io::save_bytes(&wb, "book.xlsx").unwrap();
+    assert_eq!(names_of(&gridcraft_xlsx::read_xlsx(&bytes).unwrap().0), ["_gc_macro_0_0", "_gc_macro_1_0", "_gc_macro_2_0"]);
+}

crates/engine/src/pivot.rs

@@ -803,6 +803,51 @@ fn data_cols_exceed_sheet(col_groups: usize, grand_total: bool, values: usize) -
 /// Computes a PivotTable report from the workbook.
 pub fn compute(wb: &Workbook, pt: &PivotTable, pivot_sheet: usize) -> Result<Output, String> {
     let src = read_source(wb, &pt.source, pivot_sheet)?;
+    compute_from(wb, pt, &src, None)
+}
+
+/// Why Show Details turns a cell down when it isn't a value cell of a PivotTable (the front end
+/// edits that cell instead).
+pub const NOT_A_VALUE_CELL: &str = "select a value cell in a PivotTable";
+
+/// Show Details: the source list and the indices of its records behind the value cell at
+/// (`row`, `col`) of the report grid (relative to its top-left), in source order. `None` when
+/// that cell holds no value (a label, a header, a blank). An error, for any cell, when the report
+/// on the sheet is no longer what the source computes: the cell's records would be another cell's.
+pub fn detail_records(wb: &Workbook, pt: &PivotTable, pivot_sheet: usize, row: u32, col: u32) -> Result<Option<(Source, Vec<usize>)>, String> {
+    let src = read_source(wb, &pt.source, pivot_sheet)?;
+    let mut probe = Probe { at: (row as usize, col as usize), hit: None, paths: Vec::new() };
+    let out = compute_from(wb, pt, &src, Some(&mut probe))?;
+    // Excel reads its PivotTable cache; without one, the report written by the last refresh
+    // must still be what the source gives. Checked first: in a stale report even a cell that
+    // computes as a label may show a value.
+    let sh = wb.sheet(pivot_sheet).ok_or("There's no such sheet.")?;
+    let current = pt.last_range.is_some_and(|r| {
+        r.height() == out.height().max(1)
+            && r.width() == out.width.max(1)
+            && out.grid.iter().enumerate().all(|(ri, line)| {
+                line.iter().enumerate().all(|(ci, oc)| {
+                    sh.value_ref(CellRef::new(r.start.row + ri as u32, r.start.col + ci as u32)).unwrap_or(&Value::Empty) == &oc.value
+                })
+            })
+    });
+    if !current {
+        return Err("The PivotTable report is out of date. Refresh it first.".into());
+    }
+    let Some((rp, cp)) = probe.hit else { return Ok(None) };
+    let records = probe.paths.into_iter().filter(|(_, r, c)| r.starts_with(&rp) && c.starts_with(&cp)).map(|(i, _, _)| i).collect();
+    Ok(Some((src, records)))
+}
+
+/// What [`detail_records`] asks of a report computation: the row and column item paths of the
+/// value cell at `at`, and the paths of every record the filters let through.
+struct Probe {
+    at: (usize, usize),
+    hit: Option<(Vec<u32>, Vec<u32>)>,
+    paths: Vec<(usize, Vec<u32>, Vec<u32>)>,
+}
+
+fn compute_from(wb: &Workbook, pt: &PivotTable, src: &Source, mut probe: Option<&mut Probe>) -> Result<Output, String> {
     let fmt_of = |c: usize| src.formats.get(c).cloned().unwrap_or_else(|| "General".into());
     let rows: Vec<(usize, &PivotField)> = pt.rows.iter().filter_map(|f| src.col(&f.source_col).map(|c| (c, f))).collect();
     let cols: Vec<(usize, &PivotField)> = pt.columns.iter().filter_map(|f| src.col(&f.source_col).map(|c| (c, f))).collect();
@@ -914,6 +959,9 @@ pub fn compute(wb: &Workbook, pt: &PivotTable, pivot_sheet: usize) -> Result<Out
         if !col_leaves.contains(&cp[..shown]) {
             col_leaves.insert(cp[..shown].to_vec());
         }
+        if let Some(pr) = probe.as_deref_mut() {
+            pr.paths.push((ri, rp.clone(), cp.clone()));
+        }
         row_set.insert(rp);
         col_set.insert(cp);
         // Refuse a report that cannot fit before the accumulators for more column groups are allocated.
@@ -1278,6 +1326,11 @@ pub fn compute(wb: &Workbook, pt: &PivotTable, pivot_sheet: usize) -> Result<Out
                     _ => raw,
                 },
             };
+            if let Some(pr) = probe.as_deref_mut()
+                && pr.at == (r, l + j)
+            {
+                pr.hit = Some((rp.to_vec(), cp.to_vec()));
+            }
             g.set(r, l + j, shown, ck.1, 0, data_fmt.get(*vi).cloned().flatten());
         }
     }

crates/engine/src/record.rs

@@ -18,6 +18,10 @@ pub const RECORD_LIMIT: usize = 100_000;
 /// without a size cap a recording of a few opens of a large workbook keeps tens of MB alive.
 pub const RECORD_MAX_BYTES: usize = 64 * 1024 * 1024;
 
+/// Commands never recorded: recovery actions, not steps of the person's work. A recording that
+/// replayed Stop Action would stop the run it is a step of.
+pub const NOT_RECORDED: [&str; 1] = ["automate.stopAction"];
+
 /// One recorded command and where it ran.
 #[derive(Clone, Debug, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
@@ -49,22 +53,18 @@ pub struct Recorder {
 impl Recorder {
     /// Adds a step, or counts it as dropped once the buffer is full.
     pub fn push(&mut self, step: RecordedStep) {
+        // Once a step has been dropped, or the step count is full, the rest are counted without
+        // being serialized (a `file.open` step holds a whole file).
+        if self.dropped > 0 || self.steps.len() >= RECORD_LIMIT {
+            self.dropped = self.dropped.saturating_add(1);
+            return;
+        }
         let size = serde_json::to_vec(&step).map_or(0, |v| v.len());
-        let fits = self.steps.len() < RECORD_LIMIT && self.bytes.saturating_add(size) <= RECORD_MAX_BYTES;
-        if fits && self.dropped == 0 {
+        if self.bytes.saturating_add(size) <= RECORD_MAX_BYTES {
             self.bytes = self.bytes.saturating_add(size);
             self.steps.push(step);
         } else {
             self.dropped = self.dropped.saturating_add(1);
         }
     }
-
-    /// Forgets the steps after the first `steps` and puts the dropped count back to `dropped`,
-    /// for a command the caller received as failed after it was recorded.
-    pub fn truncate(&mut self, steps: usize, dropped: usize) {
-        for step in self.steps.drain(steps.min(self.steps.len())..) {
-            self.bytes = self.bytes.saturating_sub(serde_json::to_vec(&step).map_or(0, |v| v.len()));
-        }
-        self.dropped = dropped;
-    }
 }

crates/engine/src/sample.rs

@@ -26,7 +26,7 @@ pub fn build(name: &str) -> Option<Workbook> {
             _ => return None,
         };
     }
-    s.add_document(DocState::new(wb, None, title(name)));
+    s.add_document(DocState::new(wb, None, title(name), crate::Trust::Local));
     let ok = match name {
         "budget" => budget(&mut s),
         "sales" => sales(&mut s),

crates/engine/src/tests.rs

@@ -3,6 +3,11 @@ use serde_json::json;
 
 use crate::Session;
 
+thread_local! {
+    /// Copies of a document's undo and redo history `execute` has taken on this thread.
+    pub(crate) static HISTORY_COPIES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
+}
+
 fn s() -> Session {
     let mut s = Session::new();
     s.new_workbook();

crates/engine/src/tests/seams.rs

@@ -643,6 +643,142 @@ fn a_group_ended_by_a_command_that_then_panics_keeps_its_step() {
     }
 }
 
+#[test]
+fn a_command_that_fails_rolls_back_the_groups_it_opened() {
+    // A `?` between `group_begin` and `group_end` used to leave the group open for good: Undo
+    // and Redo refused, and later edits made no steps.
+    crate::cmd::register_undoable_test_command("test.groupThenFail", |s, _| {
+        let _token = s.group_begin("Half")?;
+        put(s, "A1", "half done")?;
+        Err(EngineError::Other("stopped halfway".into()))
+    });
+    crate::cmd::register_undoable_test_command("test.editGroupThenFail", |s, _| {
+        put(s, "B1", "outside")?;
+        let _token = s.group_begin("Half")?;
+        put(s, "B2", "inside")?;
+        Err(EngineError::Other("stopped halfway".into()))
+    });
+    let mut s = s();
+    set(&mut s, "A1", "before");
+    set(&mut s, "A2", "undone");
+    s.execute("edit.undo", json!({})).unwrap();
+    s.execute("selection.set", json!({"range": "C3"})).unwrap();
+    let uid = s.doc().unwrap().uid;
+    let err = s.execute("test.groupThenFail", json!({})).unwrap_err().to_string();
+    assert!(err.contains("stopped halfway"), "{err}");
+    assert!(!s.in_group(uid), "no group is left open");
+    assert_eq!(v(&s, "A1").display(), "before");
+    assert_eq!(s.doc().unwrap().selection.a1(), "C3");
+    assert_eq!(undo_labels(&s), ["Enter Cell"]);
+    assert_eq!(s.doc().unwrap().redo.len(), 1, "the redo history comes back");
+    assert_eq!(s.execute("history.list", json!({})).unwrap()["grouping"], json!(false));
+    set(&mut s, "A3", "next");
+    assert_eq!(undo_labels(&s), ["Enter Cell", "Enter Cell"], "later edits make steps again");
+    s.execute("edit.undo", json!({})).unwrap();
+    assert!(v(&s, "A3").is_empty());
+
+    // What the command changed before opening the group stays, as for any failed command, and is
+    // a step of its own; what it changed inside the group goes.
+    assert!(s.execute("test.editGroupThenFail", json!({})).is_err());
+    assert!(!s.in_group(uid));
+    assert_eq!((v(&s, "B1").display(), v(&s, "B2").display()), ("outside".into(), String::new()));
+    assert_eq!(undo_labels(&s), ["Enter Cell", "test.editGroupThenFail"]);
+    s.execute("edit.undo", json!({})).unwrap();
+    assert!(v(&s, "B1").is_empty());
+
+    // A group open before the command stays open: only the command's own group goes.
+    let token = s.group_begin("Outer").unwrap();
+    set(&mut s, "D1", "outer");
+    assert!(s.execute("test.groupThenFail", json!({})).is_err());
+    assert!(s.in_group(uid), "the outer group is still running");
+    assert_eq!((v(&s, "A1").display(), v(&s, "D1").display()), ("before".into(), "outer".into()));
+    assert!(s.group_end(token).unwrap());
+    assert_eq!(undo_labels(&s), ["Enter Cell", "Outer"]);
+    s.execute("edit.undo", json!({})).unwrap();
+    assert!(v(&s, "D1").is_empty());
+}
+
+#[test]
+fn stop_action_rolls_back_a_group_left_open() {
+    // A host that dropped its token, or a debugger the person wants to abandon: the message on
+    // Undo names the command that stops it, and the command puts the start back.
+    let mut s = s();
+    set(&mut s, "A1", "start");
+    set(&mut s, "A2", "undone");
+    s.execute("edit.undo", json!({})).unwrap();
+    let uid = s.doc().unwrap().uid;
+    let token = s.group_begin("Debug").unwrap();
+    set(&mut s, "A1", "step 1");
+    drop(s.group_begin("Inner").unwrap());
+    set(&mut s, "A3", "step 2");
+    let err = s.execute("edit.undo", json!({})).unwrap_err().to_string();
+    assert!(err.contains("automate.stopAction"), "{err}");
+    assert_eq!(s.execute("automate.stopAction", json!({})).unwrap(), json!({"stopped": true}));
+    assert!(!s.in_group(uid));
+    assert_eq!(v(&s, "A1").display(), "start");
+    assert!(v(&s, "A3").is_empty());
+    assert_eq!(undo_labels(&s), ["Enter Cell"], "stopping is not a step of its own");
+    assert_eq!(s.doc().unwrap().redo.len(), 1, "the redo history comes back");
+    assert!(s.group_end(token).is_err(), "the token is spent");
+    assert_eq!(s.execute("automate.stopAction", json!({})).unwrap(), json!({"stopped": false}));
+    s.execute("edit.redo", json!({})).unwrap();
+    assert_eq!(v(&s, "A2").display(), "undone");
+    // Other documents keep their groups.
+    let other = s.new_workbook();
+    let other_uid = s.documents()[other].uid;
+    let _kept = s.group_begin("Theirs").unwrap();
+    s.set_active(0);
+    assert_eq!(s.execute("automate.stopAction", json!({})).unwrap(), json!({"stopped": false}));
+    assert!(s.in_group(other_uid));
+}
+
+fn history_of(s: &Session) -> Vec<(String, *const gridcraft_model::Workbook)> {
+    let d = s.doc().unwrap();
+    d.undo.iter().chain(d.redo.iter()).map(|e| (e.label.clone(), std::sync::Arc::as_ptr(&e.wb))).collect()
+}
+
+#[test]
+fn queries_take_no_copy_of_the_history() {
+    // `execute` copies the history so a panic can put it back; a query leaves it alone, so the
+    // copy is skipped for the queries front ends run all the time.
+    let mut s = s();
+    for i in 0..50 {
+        set(&mut s, "A1", &i.to_string());
+    }
+    s.execute("edit.undo", json!({})).unwrap();
+    super::HISTORY_COPIES.with(|n| n.set(0));
+    for _ in 0..100 {
+        s.execute("history.list", json!({})).unwrap();
+        s.execute("cell.get", json!({"cell": "A1"})).unwrap();
+        s.execute("document.inspect", json!({})).unwrap();
+    }
+    assert_eq!(super::HISTORY_COPIES.with(|n| n.get()), 0);
+    set(&mut s, "A2", "x");
+    assert_eq!(super::HISTORY_COPIES.with(|n| n.get()), 1, "a command that can change it still copies it");
+}
+
+#[test]
+fn every_query_leaves_the_history_alone() {
+    // What lets a query skip the copy: none of them changes the undo or redo history.
+    let mut s = Session::new();
+    s.execute("file.new", json!({"sample": "sales"})).unwrap();
+    set(&mut s, "A1", "1");
+    set(&mut s, "A1", "2");
+    s.execute("edit.undo", json!({})).unwrap();
+    let before = history_of(&s);
+    assert_eq!(before.len(), 2);
+    let mut queries = 0;
+    for spec in crate::command_specs().iter().filter(|c| !c.journal && !c.undoable) {
+        queries += 1;
+        for params in [json!({}), json!({"range": "A1:B3", "cell": "A1", "sheet": 0}), json!([1, 2])] {
+            let _ = s.execute(spec.id, params.clone());
+            s.set_active(0);
+            assert_eq!(history_of(&s), before, "{} {params}", spec.id);
+        }
+    }
+    assert!(queries > 40, "{queries}");
+}
+
 // ---------------------------------------------------------------- recording buffer
 
 fn recorded_ids(s: &Session) -> Vec<String> {
@@ -662,6 +798,74 @@ fn a_headless_call_that_fails_for_want_of_a_file_is_not_recorded() {
     assert_eq!(rec.dropped, 0);
 }
 
+#[test]
+fn a_call_that_only_opens_a_dialog_is_not_recorded() {
+    // Menu-style calls with no parameters only queue their dialog; the dialog's confirming call
+    // is the step, so playing the recording back runs each command once, with its parameters.
+    let mut s = s();
+    s.start_recording();
+    for id in ["file.open", "data.fromTextCsv", "home.formatCells"] {
+        s.execute(id, json!({})).unwrap_or_else(|e| panic!("{id}: {e}"));
+        let dialogs = s.take_ui_requests();
+        assert!(dialogs.iter().any(|q| matches!(q, crate::UiRequest::Dialog(..))), "{id} opens a dialog: {dialogs:?}");
+    }
+    assert!(recorded_ids(&s).is_empty(), "{:?}", recorded_ids(&s));
+    set(&mut s, "A1", "1");
+    s.execute("home.formatCells", json!({"range": "A1", "style": {"bold": true}})).unwrap();
+    assert_eq!(recorded_ids(&s), ["cell.set", "home.formatCells"]);
+    // A dialog queued before the call, and not taken yet, does not hide the call.
+    s.execute("file.open", json!({})).unwrap();
+    set(&mut s, "A2", "2");
+    assert_eq!(recorded_ids(&s), ["cell.set", "home.formatCells", "cell.set"]);
+    assert_eq!(s.stop_recording().unwrap().dropped, 0);
+}
+
+#[test]
+fn stop_action_runs_only_on_its_own_and_is_not_recorded() {
+    // A step of an action in progress may not stop the action: nested, the command is refused
+    // and the action's group stays. Nor is Stop Action a step of the person's work: a recording
+    // leaves it out, so playing the recording back never stops the run it is part of.
+    let mut s = s();
+    s.start_recording();
+    let r: crate::Result<()> = s.grouped("Outer", |s| {
+        set(s, "A1", "one");
+        let err = s.execute_nested("automate.stopAction", json!({})).unwrap_err().to_string();
+        assert!(err.contains("on its own"), "{err}");
+        set(s, "A2", "two");
+        Ok(())
+    });
+    r.unwrap();
+    assert_eq!(v(&s, "A2").display(), "two");
+    assert_eq!(undo_labels(&s), ["Outer"], "the action kept its group");
+    // On its own it stops a group left open; the recording keeps the edits and not the stop.
+    let _left = s.group_begin("Left open").unwrap();
+    set(&mut s, "A3", "three");
+    assert_eq!(s.execute("automate.stopAction", json!({})).unwrap(), json!({"stopped": true}));
+    assert!(v(&s, "A3").is_empty());
+    assert_eq!(recorded_ids(&s), ["cell.set", "cell.set", "cell.set"]);
+    assert_eq!(s.stop_recording().unwrap().dropped, 0);
+}
+
+#[test]
+fn a_run_that_edits_and_opens_a_dialog_is_recorded() {
+    // A script with an edit before a menu-style step: the edit happened, so the run is a step of
+    // the recording (the person's confirming call is another). Only a call that opened a dialog
+    // and changed nothing is left out.
+    let mut s = s();
+    let edit_then_dialog = json!([{"command": "cell.set", "params": {"cell": "B1", "input": "1"}}, {"command": "home.formatCells", "params": {}}]);
+    let dialog_only = json!([{"command": "home.formatCells", "params": {}}]);
+    s.execute("automate.newScript", json!({"name": "Edit", "commands": edit_then_dialog})).unwrap();
+    s.execute("automate.newScript", json!({"name": "Ask", "commands": dialog_only})).unwrap();
+    s.start_recording();
+    s.execute("automate.runScript", json!({"name": "Ask"})).unwrap();
+    assert!(s.take_ui_requests().iter().any(|q| matches!(q, crate::UiRequest::Dialog(..))));
+    assert!(recorded_ids(&s).is_empty(), "{:?}", recorded_ids(&s));
+    s.execute("automate.runScript", json!({"name": "Edit"})).unwrap();
+    assert!(s.take_ui_requests().iter().any(|q| matches!(q, crate::UiRequest::Dialog(..))));
+    assert_eq!(recorded_ids(&s), ["automate.runScript"], "the edit happened, so the run is a step");
+    assert_eq!(s.stop_recording().unwrap().dropped, 0);
+}
+
 #[test]
 fn the_recorder_keeps_top_level_calls_with_where_they_ran() {
     let mut s = s();
@@ -803,6 +1007,22 @@ fn documents_know_where_they_came_from() {
     assert_eq!(trust(&s), Trust::FromFile);
 }
 
+#[test]
+fn whoever_makes_a_document_says_where_it_came_from() {
+    // No default trust: a document built from bytes outside `file.open` (a recovery copy, a
+    // template) is untrusted because its maker says so, not trusted because it forgot to.
+    let mut s = Session::new();
+    let bytes = gridcraft_xlsx::write_xlsx(&gridcraft_model::Workbook::new()).unwrap();
+    let (wb, _) = crate::io::open_bytes("recovered.xlsx", &bytes).unwrap();
+    s.add_document(crate::DocState::new(wb, None, "Recovered".into(), Trust::FromFile));
+    assert_eq!(trust(&s), Trust::FromFile);
+    assert_eq!(s.execute("document.inspect", json!({})).unwrap()["trust"], "fromFile");
+    s.add_document(crate::DocState::new(gridcraft_model::Workbook::new(), None, "Made here".into(), Trust::Local));
+    assert_eq!(trust(&s), Trust::Local);
+    s.new_workbook();
+    assert_eq!(trust(&s), Trust::Local);
+}
+
 #[test]
 fn pasting_from_a_file_carries_its_trust() {
     let mut s = s(); // document 0: Local

crates/engine/tests/it/dynamicmetadata.rs

@@ -26,7 +26,7 @@ fn opened() -> Session {
     let bytes = io::save_bytes(&wb, "dynamic.xlsx").unwrap();
     let (back, _) = io::open_bytes("dynamic.xlsx", &bytes).unwrap();
     let mut s = Session::new();
-    s.add_document(DocState::new(back, None, "dynamic".into()));
+    s.add_document(DocState::new(back, None, "dynamic".into(), gridcraft_engine::Trust::FromFile));
     s
 }
 

crates/engine/tests/it/long_operator_chains.rs

@@ -78,7 +78,7 @@ fn longest_flat_chains_survive_editing_saving_and_reopening() {
         let bytes = io::save_bytes(wb, "chains.xlsx").expect("save");
         let (reopened, _) = io::open_bytes("chains.xlsx", &bytes).expect("open");
         let mut session = Session::new();
-        session.add_document(DocState::new(reopened, None, "chains".into()));
+        session.add_document(DocState::new(reopened, None, "chains".into(), gridcraft_engine::Trust::FromFile));
         session.execute("formulas.calculateNow", json!({})).expect("recalc");
         assert_eq!(value(&session, "C2"), Value::Number(4096.0));
         assert_eq!(value(&session, "C3"), Value::Number(-4094.0));

crates/engine/tests/it/stored_list_shape.rs

@@ -6,7 +6,8 @@ const NOT_OBJECTS: [&str; 4] = ["\"[1,2]\"", "\"7\"", "\"true\"", "\"\"\"hi\"\"\
 fn with_name(name: &str, refers_to: &str) -> Session {
     let mut s = Session::new();
     s.new_workbook();
-    s.execute("formulas.defineName", json!({"name": name, "refersTo": refers_to})).unwrap();
+    // `force`: the name commands refuse GridCraft's own data names otherwise.
+    s.execute("formulas.defineName", json!({"name": name, "refersTo": refers_to, "force": true})).unwrap();
     s
 }