diff --git a/ROADMAP.md b/ROADMAP.md index 7075e4f29..fb9985c72 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,7 +2,7 @@ **Stage: alpha** · next: beta, ~25 points of "ready for real work" and ~160–240 h away -> **Last reviewed:** 2026-11-01 · **Last updated:** 2026-11-02 · **Change:** minor (equations: Insert › Equation, editor, gridcraft-math; earlier: What-if and analysis: Solver Sensitivity report for Simplex LP and GRG Nonlinear; earlier: floating objects: one selection, eight resize handles, keyboard delete; earlier: Chart data tables: live source values beneath column, bar, line, area and compatible combo charts; legend keys and borders; XLSX round-trip and print/PDF; earlier: Formulas › Function Library as labelled buttons with category menus; Recently Used and Insert Function categories; XLSX workbook-view selection, bounded tab ratios and sheet-edit regressions; readiness estimates unchanged; earlier: CSV and text open detects each column's date order, so day-first dates are not silently swapped) · **Target:** Microsoft Excel (Microsoft 365) +> **Last reviewed:** 2026-11-01 · **Last updated:** 2026-11-02 · **Change:** minor (Python cells stored and kept in files; earlier: equations: Insert › Equation, editor, gridcraft-math; earlier: What-if and analysis: Solver Sensitivity report for Simplex LP and GRG Nonlinear; earlier: floating objects: one selection, eight resize handles, keyboard delete; earlier: Chart data tables: live source values beneath column, bar, line, area and compatible combo charts; legend keys and borders; XLSX round-trip and print/PDF; earlier: Formulas › Function Library as labelled buttons with category menus; Recently Used and Insert Function categories; XLSX workbook-view selection, bounded tab ratios and sheet-edit regressions; readiness estimates unchanged; earlier: CSV and text open detects each column's date order, so day-first dates are not silently swapped) · **Target:** Microsoft Excel (Microsoft 365) GridCraft aims at full Microsoft Excel parity, and at being better in places: faster, open (XLSX is the native format), fully scriptable by agents (MCP, control channel, CLI), and on the web. @@ -135,6 +135,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | Date | What landed | |---|---| +| 2026-11-02 | Python cells as data: `python.set` stores code in a cell, Python cells copy, move, fill and undo like formulas, and `.xlsx` reads and writes Python cells as [MS-XLSX] specifies (`_xlfn._xlws.PY(index,0 or 1)`, the code in the workbook's Python part) with cached values and spills, so such workbooks open with their results; nothing runs Python (new code shows `#BLOCKED!`), and opening a file limits code size, result size and spill ranges | | 2026-11-02 | Equations: Insert › Equation (a split button with a gallery of examples) and an editor with quick-insert buttons, the error underlined at its position and a live preview; `insert.equation` and `equation.edit`; LaTeX laid out by RaTeX inside the new `gridcraft-math` crate, capped and panic-guarded, and cached on the shape so that painting never runs the layout engine; xlsx writes Office Math with a plain-text fallback and the source in `descr`, and reads Office Math back into LaTeX; PDF prints equations with the standard Symbol font and `[?]` for anything else | | 2026-11-02 | Solver writes a Sensitivity report: Simplex LP shadow prices, reduced costs and allowable ranges from the final tableau; GRG Nonlinear reduced gradients and Lagrange multipliers from difference gradients, at a Karush-Kuhn-Tucker point only; Solver Results offers it where it applies (no score change) | | 2026-11-02 | Floating objects: charts, pictures, shapes and slicers share one selection with a frame of eight resize handles (each moves only its own edges, also on right-to-left sheets); Delete or Backspace deletes the selected object and Escape drops the selection; slicers follow row and column edits and sorts like the other objects; chart series follow cut-moves and sheet renames and deletes; a copied sheet gives its objects fresh ids; `ui.inspect` reports `selectedObject` and `ui.object.select` selects any object (#190, #213) | @@ -254,6 +255,7 @@ Ranked; estimates in Opus 5.5 agent-hours. Detail and the full beta-gate list in | Date | Change | Summary | |---|---|---| +| 2026-11-02 | minor | Progress log: Python cells as data | | 2026-11-02 | minor | Progress log: equations (Insert › Equation, the editor, gridcraft-math) | | 2026-11-02 | minor | What-if and analysis: Solver Sensitivity report for Simplex LP and GRG Nonlinear (no score change) | | 2026-11-02 | minor | Progress log: one selection and eight resize handles for every floating object | diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index eb11045db..63ac533de 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -135,6 +135,10 @@ impl Calc { if cell.value == Value::Error(CellError::Circ) { circ.push((si, c)); } + } else if cell.py.is_some() { + // A Python cell lays out its kept result, so it is calculated like a formula + // that reads nothing: its spill is blocked and unblocked like a formula's. + self.graph.insert((si, c), Node { areas: Vec::new(), dynamic: false }); } } } @@ -206,11 +210,29 @@ impl Calc { let mut cleared: Vec<(usize, RangeRef)> = Vec::new(); for &k in changed { let formula = wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.formula.clone()); + let py = wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.py.as_ref()); + let python = py.is_some(); + // A Python cell whose result (if any) is not an array spills nothing. + let spills = py.and_then(|p| p.result_value()).is_some_and(|v| matches!(v, Value::Array(_))); match formula.and_then(|f| f.expr()) { Some(e) => { let (areas, dynamic) = precedents(wb, k.0, &e); self.graph.insert(k, Node { areas, dynamic }); } + // See `rebuild`: its spill is laid out again from its result when it is calculated. + None if python => { + self.graph.insert(k, Node { areas: Vec::new(), dynamic: false }); + // New code has no result yet: the spill the cell had (its old result's, or a + // formula's) goes, and what read it is recalculated, as for a formula + // replaced by a constant. + if !spills + && let Some(sh) = wb.sheet_mut(k.0) + && let Some(r) = sh.spill_ranges.remove(&k.1) + { + sh.clear_spill(r); + cleared.push((k.0, r)); + } + } None => { self.graph.remove(k); // A formula replaced by a constant (or cleared) takes its spilled values with it. @@ -338,11 +360,15 @@ impl Calc { exprs.insert(*k, e); } } + let python: HashMap = dirty + .iter() + .filter_map(|k| wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.py.as_ref()).and_then(|p| p.result_value()).map(|v| (*k, v))) + .collect(); let now = self.now(); let mut rng = self.rng; let tp = prof_now(); let (results, spills, cycles) = { - let mut host = PassHost::new(wb, &exprs, &mut rng, now); + let mut host = PassHost::new(wb, &exprs, &mut rng, now).with_python(python); host.iterate = wb.calc.iterative; for k in &dirty { host.calculate(*k); @@ -547,6 +573,48 @@ pub(crate) mod tests { v } + /// A 1,000 x 1,000 Python result laid out from A1 in a pass, with the same values already + /// spilled on the sheet from the last calculation, as a full recalculation finds them. + fn big_python_spill() -> (Workbook, HashMap) { + let n = 1_000usize; + let data: Vec = (0..n * n).map(|i| Value::Number(i as f64)).collect(); + let array = gridcraft_core::Array::new(n, n, data.clone()).unwrap(); + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + let anchor = CellRef::new(0, 0); + let py = gridcraft_model::PyCode::new("big"); + sh.set_cell(anchor, Cell { value: Value::Number(0.0), py: Some(std::sync::Arc::new(py)), ..Cell::default() }); + let area = RangeRef::new(anchor, CellRef::new(n as u32 - 1, n as u32 - 1)); + sh.spill_ranges.insert(anchor, area); + for (i, c) in area.iter().enumerate().skip(1) { + sh.spill.insert(c, data[i].clone()); + } + let mut python = HashMap::default(); + python.insert((0, anchor), Value::Array(std::sync::Arc::new(array))); + (wb, python) + } + + #[test] + fn a_read_beside_a_large_spill_walks_none_of_it() { + let (wb, python) = big_python_spill(); + let exprs = HashMap::default(); + let mut rng = 1u64; + let mut host = PassHost::new(&wb, &exprs, &mut rng, 0.0).with_python(python); + assert_eq!(host.cell_value(0, CellRef::new(0, 0)), Value::Number(0.0)); + // Beside the spill (column ALM, the 1,001st), over the rows it covers. + let beside = RangeRef::parse("ALM1:ALM1000").unwrap(); + host.spill_visits = 0; + assert!(host.range_values(0, beside).iter().all(Value::is_empty)); + assert!(host.visit_range(0, beside, &mut |_| true), "a disjoint read takes the fast path"); + assert_eq!(host.spill_visits, 0, "nothing of the spill is walked"); + // A read inside it still sees the spilled values, walking only the cells it reads. + let inside = RangeRef::parse("B2:C3").unwrap(); + host.spill_visits = 0; + assert_eq!(host.range_values(0, inside), [1_001.0, 1_002.0, 2_001.0, 2_002.0].map(Value::Number).to_vec()); + assert!(host.spill_visits <= 8, "{}", host.spill_visits); + assert!(!host.visit_range(0, inside, &mut |_| true), "an overlapping read leaves the block to range_values"); + } + #[test] fn intervals_match_a_linear_scan() { let mut iv: Intervals = Intervals::default(); diff --git a/crates/calc/src/recalc/pass_host.rs b/crates/calc/src/recalc/pass_host.rs index 6aa8a1204..5909369fd 100644 --- a/crates/calc/src/recalc/pass_host.rs +++ b/crates/calc/src/recalc/pass_host.rs @@ -45,6 +45,10 @@ pub(super) struct PassHost<'a> { /// Suspensions so far (tests check that a wide read at the limit needs one, not one per cell). #[cfg(test)] pub(super) suspensions: usize, + /// Spilled values looked at by range reads (tests check that a read outside a large spill + /// looks at none of it). + #[cfg(test)] + pub(super) spill_visits: usize, /// The formulas in `cycles`. cycle_seen: HashSet, /// Iterative calculation: a circular reference reads the cell's previous value instead of @@ -64,6 +68,10 @@ pub(super) struct PassHost<'a> { stack_top: usize, /// Whether an evaluation ran past the stack budget since the innermost formula started. out_of_stack: bool, + /// The kept results of the Python cells in the pass: a Python cell isn't evaluated, its + /// result is laid out (an array spills from the cell, as a formula's does). Each one is in + /// `pending` until it is laid out, so a formula that reads it first gets its new value. + python: HashMap, } impl<'a> PassHost<'a> { @@ -87,6 +95,8 @@ impl<'a> PassHost<'a> { also: Vec::new(), #[cfg(test)] suspensions: 0, + #[cfg(test)] + spill_visits: 0, cycle_seen: HashSet::default(), iterate: false, prev: HashMap::default(), @@ -96,8 +106,17 @@ impl<'a> PassHost<'a> { taint: false, stack_top: 0, out_of_stack: false, + python: HashMap::default(), } } + + /// Adds the dirty Python cells of the pass and their kept results: like the formulas, they + /// are pending until laid out, so every read path calculates them before reading them. + pub(super) fn with_python(mut self, python: HashMap) -> Self { + self.pending.extend(python.keys().copied()); + self.python = python; + self + } } /// Whether an array result at `k` can't spill: it is in a table, or its area runs off the sheet, @@ -111,7 +130,10 @@ fn spill_blocked(host: &PassHost<'_>, (sheet, anchor): Key, a: &Array) -> bool { let own = sh.spill_ranges.get(&anchor); let spilled_over = |c: CellRef| { !own.is_some_and(|o| o.contains(c)) - && sh.spill_ranges.iter().any(|(o, r)| *o != anchor && r.contains(c) && !host.exprs.contains_key(&(sheet, *o))) + && sh.spill_ranges.iter().any(|(o, r)| { + // A formula or Python cell recalculated in this pass lays its array out again. + *o != anchor && r.contains(c) && !host.exprs.contains_key(&(sheet, *o)) && !host.python.contains_key(&(sheet, *o)) + }) }; end.row >= gridcraft_core::MAX_ROWS || end.col >= gridcraft_core::MAX_COLS @@ -135,6 +157,42 @@ pub(super) fn spills_as_before(sheet: &gridcraft_model::Sheet, anchor: CellRef, .all(|(i, c)| c == anchor || (!sheet.cells.has(c) && sheet.spill.get(&c) == Some(arr.data.get(i).unwrap_or(&Value::Empty)))) } +/// The area an array anchored at `anchor` covers. +fn array_area(anchor: CellRef, a: &Array) -> RangeRef { + let end = + CellRef::new(anchor.row.saturating_add((a.rows as u32).saturating_sub(1)), anchor.col.saturating_add((a.cols as u32).saturating_sub(1))); + RangeRef::new(anchor, end) +} + +/// The spilled values `sheet` stores inside `range`, passed to `f` until it returns false. Values +/// outside the range are not walked: with fewer rows in the range than spilled values, each row +/// is looked up on its own; otherwise one ordered scan visits each spilled value at most once. +/// Returns how many values were looked at. +fn stored_spill_in<'s>(sheet: &'s gridcraft_model::Sheet, range: RangeRef, f: &mut dyn FnMut(CellRef, &'s Value) -> bool) -> usize { + let mut seen = 0usize; + if sheet.spill.is_empty() { + return 0; + } + if (range.height() as usize) < sheet.spill.len() { + for row in range.start.row..=range.end.row { + for (c, v) in sheet.spill.range(CellRef::new(row, range.start.col)..=CellRef::new(row, range.end.col)) { + seen += 1; + if !f(*c, v) { + return seen; + } + } + } + } else { + for (c, v) in sheet.spill.range(range.start..=range.end) { + seen += 1; + if range.contains(*c) && !f(*c, v) { + return seen; + } + } + } + seen +} + impl PassHost<'_> { /// Calculates `k` (and the formulas it reads) with bounded stack: when [`Self::compute`] would /// go deeper than [`MAX_NESTED`], it suspends what is in progress, the formula it needed is @@ -188,6 +246,9 @@ impl PassHost<'_> { } return v; } + if let Some(result) = self.python.get(&k).cloned() { + return self.python_result(k, result); + } let Some(expr) = self.exprs.get(&k) else { return self.wb.sheet(k.0).map(|s| s.value(k.1)).unwrap_or_default(); }; @@ -305,6 +366,36 @@ impl PassHost<'_> { v } + /// A Python cell's kept result as its value: an array spills from the cell unless it is + /// blocked, malformed or larger than [`gridcraft_model::python::MAX_RESULT_CELLS`] (only a + /// crafted file holds one), which shows `#SPILL!`. + fn python_result(&mut self, k: Key, result: Value) -> Value { + self.pending.remove(&k); + let v = match result { + Value::Array(a) => { + let sane = a.rows >= 1 + && a.cols >= 1 + && a.rows.checked_mul(a.cols).is_some_and(|n| n == a.data.len() && n <= gridcraft_model::python::MAX_RESULT_CELLS); + if sane { + self.spills.insert(k, a.clone()); + a.data.first().cloned().unwrap_or_default() + } else { + Value::Error(CellError::Spill) + } + } + v => v, + }; + let v = match self.spills.get(&k) { + Some(a) if spill_blocked(self, k, a) => { + self.spills.remove(&k); + Value::Error(CellError::Spill) + } + _ => v, + }; + self.results.insert(k, v.clone()); + v + } + /// A formula's previous value for iterative calculation: from the last iteration, else the /// value it shows. Blank, and the `#CIRC!` of a loop calculated with iteration off, are 0. fn previous(&self, k: Key) -> Value { @@ -445,26 +536,29 @@ impl Host for PassHost<'_> { let (h, w) = (range.height() as usize, range.width() as usize); let mut out = vec![Value::Empty; h * w]; let Some(sh) = self.wb.sheet(sheet) else { return out }; - // Sparse walk: only stored cells; formula cells that are (or were) dirty go through - // `compute`, constants are copied straight from the store. + // Sparse walk: only stored cells; formula and Python cells that are (or were) dirty go + // through `compute`, constants are copied straight from the store. let mut formulas: Vec<(usize, CellRef)> = Vec::new(); for (c, cell) in sh.cells.iter_range(range) { let i = (c.row - range.start.row) as usize * w + (c.col - range.start.col) as usize; - if cell.formula.is_some() { + if cell.formula.is_some() || cell.py.is_some() { formulas.push((i, c)); } else if let Some(slot) = out.get_mut(i) { *slot = cell.value.clone(); } } - for (c, v) in sh.spill.range(range.start..=range.end) { - if range.contains(*c) { - let i = (c.row - range.start.row) as usize * w + (c.col - range.start.col) as usize; - if let Some(slot) = out.get_mut(i) - && slot.is_empty() - { - *slot = v.clone(); - } + let _seen = stored_spill_in(sh, range, &mut |c, v| { + let i = (c.row - range.start.row) as usize * w + (c.col - range.start.col) as usize; + if let Some(slot) = out.get_mut(i) + && slot.is_empty() + { + *slot = v.clone(); } + true + }); + #[cfg(test)] + { + self.spill_visits += _seen; } for (i, c) in formulas { let v = self.cell_value(sheet, c); @@ -473,26 +567,31 @@ impl Host for PassHost<'_> { } } if !self.spills.is_empty() { - let new: Vec<(usize, Value)> = self - .spills - .iter() - .filter(|(a, _)| a.0 == sheet) - .flat_map(|(a, arr)| { - let mut v = Vec::new(); - for r in 0..arr.rows { - for cc in 0..arr.cols { - let c = CellRef::new(a.1.row.saturating_add(r as u32), a.1.col.saturating_add(cc as u32)); - if c != a.1 && range.contains(c) { - v.push(( - (c.row - range.start.row) as usize * w + (c.col - range.start.col) as usize, - arr.get(r, cc).cloned().unwrap_or_default(), - )); - } + // Only the part of each array inside the range is walked: a large array beside a small + // read costs nothing. + let mut new: Vec<(usize, Value)> = Vec::new(); + for (a, arr) in self.spills.iter().filter(|(a, _)| a.0 == sheet) { + let area = array_area(a.1, arr); + if !area.intersects(&range) { + continue; + } + let (r0, r1) = (area.start.row.max(range.start.row), area.end.row.min(range.end.row)); + let (c0, c1) = (area.start.col.max(range.start.col), area.end.col.min(range.end.col)); + for row in r0..=r1 { + for col in c0..=c1 { + let c = CellRef::new(row, col); + if c == a.1 { + continue; } + let v = arr.get((row - a.1.row) as usize, (col - a.1.col) as usize).cloned().unwrap_or_default(); + new.push(((row - range.start.row) as usize * w + (col - range.start.col) as usize, v)); } - v - }) - .collect(); + } + } + #[cfg(test)] + { + self.spill_visits += new.len(); + } // Iterating: an array spilled by a formula that read a previous value. if self.iterate && self.spills.iter().any(|(a, arr)| { @@ -516,21 +615,31 @@ impl Host for PassHost<'_> { fn visit_range(&mut self, sheet: usize, range: RangeRef, f: &mut dyn FnMut(&Value) -> bool) -> bool { let wb = self.wb; let Some(sh) = wb.sheet(sheet) else { return false }; - // Formula cells first, in row-major order like `range_values`: reading one calculates it - // and the pass remembers the value, so the walk below gets the same value again without - // this method holding any. + // Formula and Python cells first, in row-major order like `range_values`: reading one + // calculates it (a Python cell lays out its result) and the pass remembers the value, so + // the walk below gets the same value again without this method holding any. for (c, cell) in sh.cells.iter_range(range) { - if cell.formula.is_some() { + if cell.formula.is_some() || cell.py.is_some() { let _ = self.cell_value(sheet, c); } } // Spilled values sit between the stored cells (and the formulas just calculated may have // produced some): leave such blocks to `range_values`. - if self.spills.keys().any(|a| a.0 == sheet) || sh.spill.range(range.start..=range.end).any(|(c, _)| range.contains(*c)) { + let arrays = self.spills.iter().any(|(a, arr)| a.0 == sheet && array_area(a.1, arr).intersects(&range)); + let mut stored = false; + let _seen = stored_spill_in(sh, range, &mut |_, _| { + stored = true; + false + }); + #[cfg(test)] + { + self.spill_visits += _seen; + } + if arrays || stored { return false; } for (c, cell) in sh.cells.iter_range(range) { - let go = if cell.formula.is_some() { f(&self.cell_value(sheet, c)) } else { f(&cell.value) }; + let go = if cell.formula.is_some() || cell.py.is_some() { f(&self.cell_value(sheet, c)) } else { f(&cell.value) }; if !go { break; } diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 89e6e7327..d63590737 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -2312,3 +2312,63 @@ fn loan_schedule_residue_keeps_excels_sign() { } assert_eq!(t.get("P7"), Value::Text("".into())); } + +/// Makes `at` a Python cell showing `cached` (its value before this calculation), with `result` +/// as its kept result, and returns the cell's key for `Calc::cells_changed`. +fn plant_python(t: &mut T, at: &str, cached: Value, result: Value) -> (usize, CellRef) { + use gridcraft_model::python::{PyCode, PyOutput}; + let output = PyOutput { value: result, type_name: "list".into(), preview: String::new(), convertible: true }; + let py = PyCode { output: Some(output), ..PyCode::new("[5, 6]") }; + let cell = Cell { value: cached, formula: None, style: gridcraft_model::StyleId::default(), py: Some(std::sync::Arc::new(py)) }; + t.wb.sheet_mut(0).unwrap().cells.set(c(at), cell); + (0, c(at)) +} + +fn column(values: &[f64]) -> Value { + let data = values.iter().map(|v| Value::Number(*v)).collect(); + Value::Array(std::sync::Arc::new(gridcraft_core::Array::new(values.len(), 1, data).unwrap())) +} + +#[test] +fn formulas_read_a_dirty_python_cell_after_it_is_laid_out() { + // B1's result can't spill (B2 holds a value), so it is #SPILL!. A1, A2 and A3 come before + // it in the pass; each must lay it out first rather than read its old value, 1. + let build = |t: &mut T| { + t.set("B2", "10"); + t.set("B3", "100"); + t.set("A1", "=B1"); + t.set("A2", "=MAX(B1:B3)"); + t.set("A3", "=SUM(B1:B3)"); + t.set("A4", "=COUNTA(B1:B3)"); + t.set("A5", "=SUMPRODUCT(--ISERROR(B1:B3))"); + }; + let check = |t: &T, how: &str| { + assert_eq!(t.get("B1"), Value::Error(CellError::Spill), "{how}"); + assert_eq!(t.get("A1"), Value::Error(CellError::Spill), "{how}: a reference"); + assert_eq!(t.get("A2"), Value::Error(CellError::Spill), "{how}: a range read"); + assert_eq!(t.get("A3"), Value::Error(CellError::Spill), "{how}: a range walked cell by cell"); + assert_eq!(t.num("A4"), 3.0, "{how}"); + assert_eq!(t.num("A5"), 1.0, "{how}: the error is seen in the range"); + }; + + // A full recalculation. + let mut t = T::new(); + build(&mut t); + plant_python(&mut t, "B1", Value::Number(1.0), column(&[5.0, 6.0])); + t.calc.recalc_all(&mut t.wb); + check(&t, "full"); + + // An incremental one: B1 becomes a Python cell after its readers. + let mut t = T::new(); + t.set("B1", "1"); + build(&mut t); + assert_eq!((t.num("A1"), t.num("A3")), (1.0, 111.0)); + let k = plant_python(&mut t, "B1", Value::Number(1.0), column(&[5.0, 6.0])); + t.calc.cells_changed(&mut t.wb, &[k]); + check(&t, "incremental"); + + // Unblocked, the readers see the new array. + t.set("B2", ""); + assert_eq!((t.num("B1"), t.num("B2")), (5.0, 6.0)); + assert_eq!((t.num("A1"), t.num("A2"), t.num("A3")), (5.0, 100.0, 111.0)); +} diff --git a/crates/core/src/value.rs b/crates/core/src/value.rs index 46721beda..0000028e9 100644 --- a/crates/core/src/value.rs +++ b/crates/core/src/value.rs @@ -21,10 +21,18 @@ pub enum CellError { Calc, /// Circular reference that could not be resolved (shown as 0 by Excel; we keep it explicit). Circ, + /// A Python cell's code raised an error, or Python stopped. + Python, + /// A Python cell is waiting to run. + Busy, + /// A Python cell ran longer than the time limit. + Timeout, + /// Python can't run here: the workbook's content isn't enabled, or this build has no Python. + Blocked, } impl CellError { - pub const ALL: [CellError; 10] = [ + pub const ALL: [CellError; 14] = [ CellError::Null, CellError::Div0, CellError::Value, @@ -35,6 +43,10 @@ impl CellError { CellError::GettingData, CellError::Spill, CellError::Calc, + CellError::Python, + CellError::Busy, + CellError::Timeout, + CellError::Blocked, ]; pub fn as_str(&self) -> &'static str { match self { @@ -49,9 +61,16 @@ impl CellError { CellError::Spill => "#SPILL!", CellError::Calc => "#CALC!", CellError::Circ => "#CIRC!", + CellError::Python => "#PYTHON!", + CellError::Busy => "#BUSY!", + CellError::Timeout => "#TIMEOUT!", + CellError::Blocked => "#BLOCKED!", } } - /// `ERROR.TYPE` number. + /// `ERROR.TYPE` number. Microsoft documents 1 to 8; 9 (#SPILL!), 11 (#BLOCKED!) and 14 (#CALC!) + /// are the numbers Excel returns. Microsoft documents no number for #PYTHON!, #BUSY! or + /// #TIMEOUT!: 19, 20 and 21 are GridCraft's own choice, after the numbers Excel is known to use, + /// and may change if Excel's numbers become known. pub fn code(&self) -> u32 { match self { CellError::Null => 1, @@ -65,6 +84,10 @@ impl CellError { CellError::Spill => 9, CellError::Calc => 14, CellError::Circ => 15, + CellError::Blocked => 11, + CellError::Python => 19, + CellError::Busy => 20, + CellError::Timeout => 21, } } pub fn parse(s: &str) -> Option { diff --git a/crates/engine/src/cmd.rs b/crates/engine/src/cmd.rs index bdb3ba8c1..77fbb1d7a 100644 --- a/crates/engine/src/cmd.rs +++ b/crates/engine/src/cmd.rs @@ -16,6 +16,7 @@ pub mod inspect; pub mod macros; pub mod pivot; pub mod print; +pub mod python; pub mod review; pub mod sheet; pub mod slicer; @@ -176,6 +177,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(spelling::specs()); v.extend(solver::specs()); v.extend(pivot::specs()); + v.extend(python::specs()); v.extend(slicer::specs()); v.extend(whatif::specs()); v.extend(forecast::specs()); @@ -522,7 +524,7 @@ fn check_arrays(old: &Workbook, new: &Workbook) -> Result<()> { continue; } for c in sh.cells.diff(&after.cells) { - let filled = after.cell(c).is_some_and(|x| x.formula.is_some() || !x.value.is_empty()); + let filled = after.cell(c).is_some_and(gridcraft_model::Cell::has_content); if !filled || !sh.spill.contains_key(&c) || same_content(sh.cell(c), after.cell(c)) { continue; } @@ -538,13 +540,18 @@ fn check_arrays(old: &Workbook, new: &Workbook) -> Result<()> { Ok(()) } -/// Two cells hold the same value or formula (formats aside). +/// Two cells hold the same value, formula or Python code (formats aside; a Python cell's result +/// is not something the person edits). fn same_content(a: Option<&gridcraft_model::Cell>, b: Option<&gridcraft_model::Cell>) -> bool { - let empty = |c: Option<&gridcraft_model::Cell>| c.is_none_or(|c| c.formula.is_none() && c.value.is_empty()); + let empty = |c: Option<&gridcraft_model::Cell>| c.is_none_or(|c| !c.has_content()); match (a, b) { (Some(a), Some(b)) => match (&a.formula, &b.formula) { (Some(x), Some(y)) => x.text == y.text && x.array == y.array, - (None, None) => a.value == b.value, + (None, None) => match (&a.py, &b.py) { + (Some(x), Some(y)) => x.code == y.code && x.object == y.object, + (None, None) => a.value == b.value, + _ => false, + }, _ => false, }, _ => empty(a) && empty(b), diff --git a/crates/engine/src/cmd/analysis.rs b/crates/engine/src/cmd/analysis.rs index dee2830d6..9cd10e50f 100644 --- a/crates/engine/src/cmd/analysis.rs +++ b/crates/engine/src/cmd/analysis.rs @@ -1788,7 +1788,7 @@ fn write_report(s: &mut Session, p: &Json, tool: &str, g: Grid) -> Result { for row in rect.start.row..=rect.end.row { for (c, cell) in sh.cells.row(row, rect.start.col, rect.end.col) { - if !cell.value.is_empty() || cell.formula.is_some() { + if cell.has_content() { occupied.push(CellRef::new(row, c)); } } @@ -1816,13 +1816,13 @@ fn write_report(s: &mut Session, p: &Json, tool: &str, g: Grid) -> Result let sh = cx.sheet_mut(si)?; for c in &occupied { let style = sh.style_id(*c); - sh.set_cell(*c, Cell { value: Value::Empty, formula: None, style }); + sh.set_cell(*c, Cell { value: Value::Empty, formula: None, style, py: None }); } let mut touched = occupied.clone(); for ((r, c), v) in &g.cells { let cell = CellRef::new(at.row + r, at.col + c); let style = pct_styles.get(&cell).copied().unwrap_or_else(|| sh.style_id(cell)); - sh.set_cell(cell, Cell { value: v.clone(), formula: None, style }); + sh.set_cell(cell, Cell { value: v.clone(), formula: None, style, py: None }); touched.push(cell); } for c in touched { diff --git a/crates/engine/src/cmd/data/tools.rs b/crates/engine/src/cmd/data/tools.rs index 2cbca17bd..c4ae43ef4 100644 --- a/crates/engine/src/cmd/data/tools.rs +++ b/crates/engine/src/cmd/data/tools.rs @@ -146,7 +146,16 @@ pub(super) fn text_to_columns(s: &mut Session, p: &Json) -> Result { } } for (c, t) in out { - let cell = crate::cmd::edit::input_to_cell(&t, sheet, c, &mut cx.wb).unwrap_or(None); + let cell = match crate::cmd::edit::input_to_cell(&t, sheet, c, &mut cx.wb) { + Ok(cell) => cell, + // `=PY(…)` doesn't make a Python cell (that takes python.set): the piece is kept + // as text rather than lost. + Err(_) if gridcraft_model::python::is_py_formula(&t) => { + let style = cx.wb.sheet(sheet).map(|sh| sh.style_id(c)).unwrap_or_default(); + Some(Cell { value: gridcraft_core::Value::text(t.as_str()), formula: None, style, py: None }) + } + Err(_) => None, + }; let shm = cx.sheet_mut(sheet)?; match cell { Some(x) => shm.set_cell(c, x), diff --git a/crates/engine/src/cmd/edit.rs b/crates/engine/src/cmd/edit.rs index 70dcf0c5a..2aa9d8fe3 100644 --- a/crates/engine/src/cmd/edit.rs +++ b/crates/engine/src/cmd/edit.rs @@ -35,7 +35,7 @@ use go_to::go_to_special; #[cfg(test)] use go_to::{GO_TO_MAX_AREAS, GO_TO_RECENT, reference_parts}; use paste_special::paste_special; -pub(crate) use paste_special::{rewrite_all_formulas, rewrite_cell_links}; +pub(crate) use paste_special::{rewrite_all_formulas, rewrite_cell_links, rewrite_python_refs}; use selection::{select_all, select_col, select_region, select_row, selection_move, selection_next, selection_set}; pub fn specs() -> Vec { @@ -132,7 +132,7 @@ pub fn specs() -> Vec { "Replace…", ["Home", "Editing", "Find & Select"], Some("Ctrl+H"), - "{what, with, matchCase?, wholeCell?, all?: true, within?: sheet|workbook}", + "{what, with, matchCase?, wholeCell?, all?: true, within?: sheet|workbook} → {replaced, skipped?} (in a Python cell the code is edited; skipped counts Python cells whose code would no longer fit)", has_doc, replace ), @@ -267,6 +267,7 @@ fn clear(s: &mut Session, p: &Json, what: &str) -> Result { for (_, cell) in cells.iter_mut() { let checkbox = cx.wb.styles.get(cell.style).num_fmt.as_str() == "checkbox"; cell.formula = None; + cell.py = None; if checkbox && uncheck_only { cell.value = Value::Bool(false); } else { diff --git a/crates/engine/src/cmd/edit/cell.rs b/crates/engine/src/cmd/edit/cell.rs index 8eb9075bf..610135682 100644 --- a/crates/engine/src/cmd/edit/cell.rs +++ b/crates/engine/src/cmd/edit/cell.rs @@ -5,6 +5,9 @@ use super::{ // ---------------------------------------------------------------- entry +/// Typing `=PY(` doesn't make a Python cell; this says what does. +pub(crate) const PY_FORMULA: &str = "Python cells are made with the python.set command (the code as `code`), not by typing =PY(…)."; + /// Whether typing `input` into a cell enters a formula: `=…`, or `+…`/`-…` followed by a name or /// a parenthesis (not a number), as Excel takes `+SUM(1)`. pub(crate) fn is_formula_input(input: &str) -> bool { @@ -21,14 +24,18 @@ pub(crate) fn is_formula_input(input: &str) -> bool { pub(crate) fn input_to_cell(input: &str, sheet: usize, at: CellRef, wb: &mut gridcraft_model::Workbook) -> Result> { let style = wb.sheet(sheet).map(|sh| sh.style_id(at)).unwrap_or_default(); if input.is_empty() { - let c = Cell { value: Value::Empty, formula: None, style }; + let c = Cell { value: Value::Empty, formula: None, style, py: None }; return Ok(if c.is_blank() { None } else { Some(c) }); } let fmt_is_text = wb.styles.get(style).num_fmt.as_str() == "@"; if fmt_is_text { - return Ok(Some(Cell { value: Value::text(input), formula: None, style })); + return Ok(Some(Cell { value: Value::text(input), formula: None, style, py: None })); } - if is_formula_input(input) { + let is_formula = is_formula_input(input); + if is_formula && gridcraft_model::python::is_py_formula(input) { + return Err(EngineError::Other(PY_FORMULA.into())); + } + if is_formula { let body = input.strip_prefix('=').unwrap_or(input); let expr = wb.name_call_case(parse_typed_formula(body)?); check_arg_counts(&expr)?; @@ -79,7 +86,7 @@ pub(crate) fn input_to_cell(input: &str, sheet: usize, at: CellRef, wb: &mut gri if input.contains('\n') && !wb.styles.get(style).align.wrap { style = wb.styles.derive(style, |s| s.align.wrap = true); } - Ok(Some(Cell { value: parsed.value, formula: None, style })) + Ok(Some(Cell { value: parsed.value, formula: None, style, py: None })) } /// Parses a formula as it is typed (without the `=`). Excel closes missing parentheses for you, @@ -147,7 +154,7 @@ pub(super) fn cell_set(s: &mut Session, p: &Json) -> Result { let sh = cx.sheet_mut(sheet)?; // The array takes the whole range: other cells in it lose their contents. for c in range.iter().skip(1).take(1_000_000) { - if let Some(old) = sh.cells.get(c).filter(|x| !x.value.is_empty() || x.formula.is_some()) { + if let Some(old) = sh.cells.get(c).filter(|x| x.has_content()) { let style = old.style; sh.set_cell(c, Cell { style, ..Cell::default() }); } @@ -239,10 +246,14 @@ pub(super) fn range_set_values(s: &mut Session, p: &Json) -> Result { value: Value::number(n.as_f64().unwrap_or(0.0)), formula: None, style: cx.wb.sheet(sheet).map(|sh| sh.style_id(at)).unwrap_or_default(), + py: None, + }), + Json::Bool(b) => Some(Cell { + value: Value::Bool(*b), + formula: None, + style: cx.wb.sheet(sheet).map(|sh| sh.style_id(at)).unwrap_or_default(), + py: None, }), - Json::Bool(b) => { - Some(Cell { value: Value::Bool(*b), formula: None, style: cx.wb.sheet(sheet).map(|sh| sh.style_id(at)).unwrap_or_default() }) - } // Text as given: no date or number reading, and a leading `'` is part of the // text rather than the marker that makes it text. A formula is still a // formula, which is what the ticket asks for and what `=` means in a cell. @@ -258,7 +269,7 @@ pub(super) fn range_set_values(s: &mut Session, p: &Json) -> Result { if text.contains('\n') && !cx.wb.styles.get(style).align.wrap { style = cx.wb.styles.derive(style, |s| s.align.wrap = true); } - Some(Cell { value: Value::Text(text.as_str().into()), formula: None, style }) + Some(Cell { value: Value::Text(text.as_str().into()), formula: None, style, py: None }) } } _ => input_to_cell(&input, sheet, at, &mut cx.wb)?, diff --git a/crates/engine/src/cmd/edit/clipboard.rs b/crates/engine/src/cmd/edit/clipboard.rs index de921226e..3a1cd0739 100644 --- a/crates/engine/src/cmd/edit/clipboard.rs +++ b/crates/engine/src/cmd/edit/clipboard.rs @@ -92,7 +92,7 @@ fn slide_together(wb: &gridcraft_model::Workbook, sheet: usize, areas: &[RangeRe for c in cells { let Some(to) = c.offset(dr, dc) else { continue }; let style = src.cell(c).map(|x| x.style).unwrap_or_default(); - dst.set_cell(to, Cell { value: src.value(c), formula: None, style }); + dst.set_cell(to, Cell { value: src.value(c), formula: None, style, py: None }); if let Some(picture) = src.cell_pictures.get(&c) { dst.cell_pictures.insert(to, picture.clone()); } diff --git a/crates/engine/src/cmd/edit/find.rs b/crates/engine/src/cmd/edit/find.rs index bce0ba728..0d4656320 100644 --- a/crates/engine/src/cmd/edit/find.rs +++ b/crates/engine/src/cmd/edit/find.rs @@ -115,6 +115,7 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { let all = bool_param(p, "all").unwrap_or(true); let workbook = str_param(p, "within") == Some("workbook"); let active = s.doc()?.selection.active; + let mut skipped = 0usize; let n = edit(s, |cx| { let sheets: Vec = if workbook { (0..cx.wb.sheets.len()).collect() } else { vec![cx.wb.active_sheet()] }; let mut n = 0; @@ -135,6 +136,30 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { } else { replace_ci(&text, &what, &with) }; + // A Python cell's text is its code: the replacement edits the code and the cell + // stays a Python cell. Code that would no longer fit in a PY formula is left as it was. + let old_py = cx.wb.sheet(si).and_then(|sh| sh.cell(c)).and_then(|x| x.py.clone()); + if let Some(old) = old_py { + let py = gridcraft_model::PyCode { object: old.object, ..gridcraft_model::PyCode::new(&new) }; + if py.code == old.code { + continue; + } + if !py.fits() { + skipped += 1; + continue; + } + // Its old spill is left to the recalculation, which removes it and + // recalculates what read it. + let sh = cx.sheet_mut(si)?; + let style = sh.style_id(c); + sh.set_cell( + c, + Cell { value: Value::Error(gridcraft_core::CellError::Blocked), formula: None, style, py: Some(std::sync::Arc::new(py)) }, + ); + cx.touch(si, c); + n += 1; + continue; + } let cell = input_to_cell(&new, si, c, &mut cx.wb).unwrap_or_else(|_| Some(Cell::value(Value::text(new.as_str())))); let sh = cx.sheet_mut(si)?; match cell { @@ -149,6 +174,9 @@ pub(super) fn replace(s: &mut Session, p: &Json) -> Result { } Ok(n) })?; + if skipped > 0 { + return Ok(json!({"replaced": n, "skipped": skipped})); + } Ok(json!({"replaced": n})) } diff --git a/crates/engine/src/cmd/edit/go_to.rs b/crates/engine/src/cmd/edit/go_to.rs index bc4f414db..0cd7a3de9 100644 --- a/crates/engine/src/cmd/edit/go_to.rs +++ b/crates/engine/src/cmd/edit/go_to.rs @@ -174,8 +174,8 @@ pub(super) fn go_to_special(s: &mut Session, p: &Json) -> Result { let v = sheet.value(c); let ok = match kind { "blanks" => v.is_empty(), - "constants" => cell.is_some_and(|x| x.formula.is_none() && !x.value.is_empty()), - "formulas" => cell.is_some_and(|x| x.formula.is_some()), + "constants" => cell.is_some_and(|x| x.formula.is_none() && x.py.is_none() && !x.value.is_empty()), + "formulas" => cell.is_some_and(|x| x.formula.is_some() || x.py.is_some()), "comments" | "notes" => sheet.comments.contains_key(&c), "errors" => v.is_error(), "numbers" => v.is_number(), diff --git a/crates/engine/src/cmd/edit/paste_special.rs b/crates/engine/src/cmd/edit/paste_special.rs index c9e296ba5..2477dc1a6 100644 --- a/crates/engine/src/cmd/edit/paste_special.rs +++ b/crates/engine/src/cmd/edit/paste_special.rs @@ -145,6 +145,7 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { } moved_formulas = taken; } + let dest_name = cx.wb.sheet(dest_sheet).map(|sh| sh.name.clone()).unwrap_or_default(); for (tr, tc, r, c) in tiled_cells(src.height(), src.width(), tiles_r, tiles_c) { let sc = CellRef::new(src.start.row + r, src.start.col + c); let (dr, dc) = if transpose { (c, r) } else { (r, c) }; @@ -152,7 +153,7 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { let src_cell = if cut_move { moved_formulas.iter().find(|(k, _)| *k == sc).map(|(_, v)| v.clone()) } else { src_sheet.cell(sc).cloned() }; let src_val = src_sheet.value(sc); let src_picture = if cut_move { moved_pictures.get(&sc) } else { src_sheet.cell_pictures.get(&sc) }.cloned(); - if skip_blanks && src_picture.is_none() && src_val.is_empty() && src_cell.as_ref().is_none_or(|x| x.formula.is_none()) { + if skip_blanks && src_picture.is_none() && src_val.is_empty() && src_cell.as_ref().is_none_or(|x| x.formula.is_none() && x.py.is_none()) { continue; } let old = cx.wb.sheet(dest_sheet).and_then(|sh| sh.cell(dest)).cloned().unwrap_or_default(); @@ -186,6 +187,33 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { let moved = gridcraft_formula::adjust::shift_relative(e, dest.row as i64 - sc.row as i64, dest.col as i64 - sc.col as i64); Some(std::sync::Arc::new(Formula { dynamic: f.dynamic, ..Formula::from_expr(moved) })) }; + // A Python cell's code travels with it. Cut to another sheet, it keeps reading the + // cells its `xl()` references named on its original sheet, as a moved formula does: + // they are qualified with that sheet before the workbook-wide move adjustment below. + let moved_py = |cell: &Cell| -> Result>> { + let Some(py) = cell.py.clone() else { return Ok(None) }; + if !cut_move || dest_sheet == clip.sheet { + return Ok(Some(py)); + } + let qualified = gridcraft_model::python::rewrite_xl(&py.code, |reference| { + let e = gridcraft_formula::parse(reference).ok()?.map(&mut |x| match x { + gridcraft_formula::Expr::Ref(mut r) if r.sheet == gridcraft_formula::SheetSel::Current => { + r.sheet = gridcraft_formula::SheetSel::Named(src_sheet_name.clone()); + gridcraft_formula::Expr::Ref(r) + } + other => other, + }); + Some(gridcraft_formula::print(&e)) + }); + Ok(Some(match qualified { + Some(code) => { + let moved = gridcraft_model::PyCode { code, ..(*py).clone() }; + check_python_fits(&moved, &dest_name, dest)?; + std::sync::Arc::new(moved) + } + None => py, + })) + }; if link { let sheet_prefix = if dest_sheet == clip.sheet && same_doc { String::new() @@ -194,6 +222,7 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { }; new.formula = Some(std::sync::Arc::new(Formula::new(&format!("={sheet_prefix}{}", sc.a1())))); new.value = Value::Empty; + new.py = None; new_picture = None; } else { let sc_cell = src_cell.clone().unwrap_or_default(); @@ -202,6 +231,7 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { new_picture = src_picture.clone(); new.formula = shift(&sc_cell); new.value = if new.formula.is_some() { Value::Empty } else { sc_cell.value.clone() }; + new.py = moved_py(&sc_cell)?; let st = map_style(sc_cell.style, &mut cx.wb); new.style = if what == What::AllExceptBorders { let borders = cx.wb.styles.get(old.style).borders; @@ -214,6 +244,7 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { new_picture = src_picture.clone(); new.formula = shift(&sc_cell); new.value = if new.formula.is_some() { Value::Empty } else { sc_cell.value.clone() }; + new.py = moved_py(&sc_cell)?; if what == What::FormulasAndNumberFormats { let nf = src_wb.styles.get(sc_cell.style).num_fmt.clone(); new.style = cx.wb.styles.derive(old.style, |s| s.num_fmt = nf); @@ -222,6 +253,7 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { What::Values | What::ValuesAndNumberFormats => { new_picture = src_picture.clone(); new.formula = None; + new.py = None; new.value = src_val.clone(); if what == What::ValuesAndNumberFormats { let nf = src_wb.styles.get(sc_cell.style).num_fmt.clone(); @@ -232,7 +264,11 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { What::Comments | What::Validation | What::ColumnWidths => {} } // Operations combine numbers with what is already there. - if op != "none" && matches!(what, What::All | What::Values | What::ValuesAndNumberFormats | What::Formulas) && new.formula.is_none() { + if op != "none" + && matches!(what, What::All | What::Values | What::ValuesAndNumberFormats | What::Formulas) + && new.formula.is_none() + && new.py.is_none() + { if old_picture.is_some() || new_picture.is_some() { new_picture = None; new.value = Value::Error(gridcraft_core::CellError::Value); @@ -292,7 +328,7 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { } else { gridcraft_formula::adjust::Edit::MoveToSheet { from: src, to_sheet: dest_name, to_row: at.row, to_col: at.col } }; - rewrite_all_formulas(&mut cx.wb, &src_sheet_name, &e); + rewrite_all_formulas(&mut cx.wb, &src_sheet_name, &e)?; cx.structural = true; } let end = at.offset_clamped((tiles_r * h) as i64 - 1, (tiles_c * w) as i64 - 1); @@ -316,7 +352,7 @@ pub(super) fn paste_special(s: &mut Session, p: &Json) -> Result { } /// Applies a reference adjustment to every cell formula and defined name in the workbook. -pub(crate) fn rewrite_all_formulas(wb: &mut gridcraft_model::Workbook, target: &str, e: &gridcraft_formula::adjust::Edit) { +pub(crate) fn rewrite_all_formulas(wb: &mut gridcraft_model::Workbook, target: &str, e: &gridcraft_formula::adjust::Edit) -> Result<()> { use gridcraft_formula::adjust::Edit; // Rows and columns inserted or deleted on the target sheet move its legacy array ranges too // (a cut moves its arrays itself). @@ -356,6 +392,7 @@ pub(crate) fn rewrite_all_formulas(wb: &mut gridcraft_model::Workbook, target: & } } rewrite_cell_links(wb, |host, expr| gridcraft_formula::adjust::adjust(expr, host, target, e)); + rewrite_python_refs(wb, &|host, expr| gridcraft_formula::adjust::adjust(expr, host, target, e)) } /// Runs `f` over every reference a floating object holds (a chart's series values, categories, @@ -385,3 +422,78 @@ pub(crate) fn rewrite_cell_links(wb: &mut gridcraft_model::Workbook, f: impl Fn( } } } + +/// Rewrites the `xl("…")` literals in every Python cell with `f`, which maps a reference held on +/// sheet `host` to its new form. Every reference-moving edit (insert and delete of lines or cells, +/// cut-move, sheet rename and delete) runs it, as it adjusts formulas. A literal is only replaced +/// when the reference it names changed, and text that doesn't parse is kept. A cell's cached +/// result is kept while its references still name the same data (a rename, a move); a reference +/// the edit deleted (`#REF!`), or one that grew or shrank (rows or columns inserted or deleted +/// inside it), leaves the result stale, so the cell is new code (`#BLOCKED!`) and the +/// recalculation removes its spill. A rewrite that would +/// make a cell's code longer than one cell holds (a longer sheet name, `Z1` becoming `AA1`) refuses +/// the whole edit, which then changes nothing: see [`check_python_fits`]. +pub(crate) fn rewrite_python_refs( + wb: &mut gridcraft_model::Workbook, + f: &impl Fn(&str, gridcraft_formula::Expr) -> gridcraft_formula::Expr, +) -> Result<()> { + for si in 0..wb.sheets.len() { + let Some(sh) = wb.sheets.get(si) else { continue }; + let host = sh.name.clone(); + let mut new_code: Vec<(CellRef, String, bool)> = Vec::new(); + for (c, cell) in sh.cells.iter() { + let Some(py) = &cell.py else { continue }; + let mut lost = false; + let changed = gridcraft_model::python::rewrite_xl(&py.code, |reference| { + let expr = gridcraft_formula::parse(reference).ok()?; + let before = gridcraft_formula::print(&expr); + let after = gridcraft_formula::print(&f(&host, expr)); + lost |= (after.contains("#REF!") && !before.contains("#REF!")) || area_shape(&before) != area_shape(&after); + (after != before).then_some(after) + }); + if let Some(code) = changed { + check_python_fits(&gridcraft_model::PyCode { code: code.clone(), object: py.object, ..Default::default() }, &host, c)?; + new_code.push((c, code, lost)); + } + } + if new_code.is_empty() { + continue; + } + let Some(sh) = wb.sheet_mut(si) else { continue }; + for (c, code, lost) in new_code { + if let Some(cell) = sh.cells.get_mut(c) + && let Some(py) = cell.py.as_mut() + { + let py = std::sync::Arc::make_mut(py); + py.code = code; + if lost { + py.output = None; + cell.value = Value::Error(gridcraft_core::CellError::Blocked); + } + } + } + } + Ok(()) +} + +/// The rows and columns a printed reference covers (`A1:B3`, `Sheet2!$A$1`, `C:E`), or `None` +/// for any other form (a name, an error). +fn area_shape(reference: &str) -> Option<(u32, u32)> { + let r = RangeRef::parse(reference.rsplit('!').next()?)?; + Some((r.height(), r.width())) +} + +/// Refuses Python code an edit has made longer than one cell holds: the limit `python.set` +/// enforces and opening a file applies (code over it is dropped to its value on open). The edit +/// is refused rather than made, so no code is lost and none keeps a reference the edit moved. +pub(crate) fn check_python_fits(py: &gridcraft_model::PyCode, sheet: &str, at: CellRef) -> Result<()> { + if py.fits() { + return Ok(()); + } + Err(EngineError::Other(format!( + "The edit would make the Python code in {}!{} longer than Excel's {}-character formula limit, so nothing was changed. Shorten the code first.", + gridcraft_formula::quote_sheet(sheet), + at.a1(), + gridcraft_model::python::MAX_FORMULA_CHARS + ))) +} diff --git a/crates/engine/src/cmd/extra/cells.rs b/crates/engine/src/cmd/extra/cells.rs index 795d7f011..838ce8342 100644 --- a/crates/engine/src/cmd/extra/cells.rs +++ b/crates/engine/src/cmd/extra/cells.rs @@ -51,6 +51,7 @@ pub(super) fn toggle_checkbox(s: &mut Session, p: &Json) -> Result { let sh = cx.sheet_mut(si)?; if let Some(cell) = sh.cells.get_mut(c) && cell.formula.is_none() + && cell.py.is_none() { // An empty checkbox cell counts as FALSE. let next = match cell.value { diff --git a/crates/engine/src/cmd/forecast.rs b/crates/engine/src/cmd/forecast.rs index ce29e9ffc..3c62e469f 100644 --- a/crates/engine/src/cmd/forecast.rs +++ b/crates/engine/src/cmd/forecast.rs @@ -315,7 +315,7 @@ fn forecast_sheet(s: &mut Session, p: &Json) -> Result { let vstyle = cx.wb.styles.derive(StyleId::DEFAULT, |st| st.num_fmt = vfmt.clone()); let mut sheet = Sheet::new(sheet_name.clone()); let put = |sheet: &mut Sheet, row: u32, col: u32, v: Value, style: StyleId| { - sheet.set_cell(CellRef::new(row, col), Cell { value: v, formula: None, style }); + sheet.set_cell(CellRef::new(row, col), Cell { value: v, formula: None, style, py: None }); }; let formula = |sheet: &mut Sheet, row: u32, col: u32, text: String, style: StyleId| { let mut cell = Cell::formula(Formula::new(&text)); diff --git a/crates/engine/src/cmd/format.rs b/crates/engine/src/cmd/format.rs index f0635d320..1066f48e2 100644 --- a/crates/engine/src/cmd/format.rs +++ b/crates/engine/src/cmd/format.rs @@ -656,8 +656,7 @@ fn merge(s: &mut Session, p: &Json, how: &str) -> Result { let others: Vec = sh.cells.iter_range(m).map(|(c, _)| c).filter(|c| *c != m.start).collect(); for c in others { if let Some(mut cell) = sh.cells.get(c).cloned() { - cell.value = gridcraft_core::Value::Empty; - cell.formula = None; + cell.clear_content(); sh.set_cell(c, cell); } touched.push((sheet, c)); @@ -674,8 +673,7 @@ fn merge(s: &mut Session, p: &Json, how: &str) -> Result { let others: Vec = sh.cells.iter_range(*r).map(|(c, _)| c).filter(|c| *c != r.start).collect(); for c in others { if let Some(mut cell) = sh.cells.get(c).cloned() { - cell.value = gridcraft_core::Value::Empty; - cell.formula = None; + cell.clear_content(); sh.set_cell(c, cell); } touched.push((sheet, c)); @@ -692,7 +690,10 @@ fn merge(s: &mut Session, p: &Json, how: &str) -> Result { let blocked = sh .cells .iter_range(reach) - .filter(|(_, x)| x.formula.is_some() && x.value == gridcraft_core::Value::Error(gridcraft_core::CellError::Spill)); + // A Python cell's result is blocked and unblocked like a formula's. + .filter(|(_, x)| { + (x.formula.is_some() || x.py.is_some()) && x.value == gridcraft_core::Value::Error(gridcraft_core::CellError::Spill) + }); touched.extend(blocked.map(|(c, _)| (sheet, c))); } cx.changed.extend(touched); diff --git a/crates/engine/src/cmd/formulas.rs b/crates/engine/src/cmd/formulas.rs index b37ad4a73..d72f814a1 100644 --- a/crates/engine/src/cmd/formulas.rs +++ b/crates/engine/src/cmd/formulas.rs @@ -635,7 +635,7 @@ fn paste_name(s: &mut Session, p: &Json) -> Result { let sh = cx.sheet_mut(sheet)?; let style = sh.style_id(c); // The second column holds the refers-to text as a text value, with no formula. - sh.set_cell(c, Cell { value: Value::text(text.as_str()), formula: None, style }); + sh.set_cell(c, Cell { value: Value::text(text.as_str()), formula: None, style, py: None }); cx.touch(sheet, c); } } diff --git a/crates/engine/src/cmd/insert.rs b/crates/engine/src/cmd/insert.rs index 59b085213..dbd4d9d78 100644 --- a/crates/engine/src/cmd/insert.rs +++ b/crates/engine/src/cmd/insert.rs @@ -316,7 +316,7 @@ fn insert_table(s: &mut Session, p: &Json) -> Result { let sh = cx.sheet_mut(sheet)?; // The header row pushes the data down one row: never off the end of the sheet. let last = RangeRef::new(CellRef::new(MAX_ROWS - 1, r.start.col), CellRef::new(MAX_ROWS - 1, r.end.col)); - if sh.cells.iter_range(last).any(|(_, x)| x.formula.is_some() || !x.value.is_empty()) { + if sh.cells.iter_range(last).any(|(_, x)| x.has_content()) { return Err(EngineError::Other("To prevent possible loss of data, cells can't be pushed off the end of the worksheet.".into())); } sh.cells.shift_rows_in_cols(r.start.col, r.end.col, r.start.row, 1); @@ -325,7 +325,7 @@ fn insert_table(s: &mut Session, p: &Json) -> Result { let e = Edit::InsertCells { axis: Axis::Rows, range: RangeRef::new(r.start, CellRef::new(r.start.row, r.end.col)) }; shift_cell_features(sh, &e); let name = sh.name.clone(); - rewrite(cx, &name, &e); + rewrite(cx, &name, &e)?; let sh = cx.sheet_mut(sheet)?; for (i, c) in (r.start.col..=r.end.col).enumerate() { sh.set_value(CellRef::new(r.start.row, c), gridcraft_core::Value::text(format!("Column{}", i + 1))); @@ -476,7 +476,10 @@ fn write_totals(sh: &mut Sheet, ti: usize) { let c = CellRef::new(row, t.range.start.col + i as u32); if let Some(code) = col.totals.subtotal_code() { let f = format!("=SUBTOTAL({code},{}[{}])", t.name, col.name); - sh.set_cell(c, Cell { formula: Some(std::sync::Arc::new(Formula::new(&f))), value: gridcraft_core::Value::Empty, style: sh.style_id(c) }); + sh.set_cell( + c, + Cell { formula: Some(std::sync::Arc::new(Formula::new(&f))), value: gridcraft_core::Value::Empty, style: sh.style_id(c), py: None }, + ); } else if let Some(l) = &col.totals_label { sh.set_value(c, gridcraft_core::Value::text(l.as_str())); } else { @@ -1323,7 +1326,7 @@ fn insert_link(s: &mut Session, p: &Json) -> Result { }); let sh = cx.sheet_mut(sheet)?; if let Some(t) = text.clone().or_else(|| sh.value(at).is_empty().then(|| target.clone())) { - sh.set_cell(at, Cell { value: gridcraft_core::Value::text(t.as_str()), formula: None, style: link_style }); + sh.set_cell(at, Cell { value: gridcraft_core::Value::text(t.as_str()), formula: None, style: link_style, py: None }); } else { sh.set_style(at, link_style); } diff --git a/crates/engine/src/cmd/insert/table.rs b/crates/engine/src/cmd/insert/table.rs index 381a9e64b..4ca624826 100644 --- a/crates/engine/src/cmd/insert/table.rs +++ b/crates/engine/src/cmd/insert/table.rs @@ -60,7 +60,7 @@ pub(super) fn insert_table(s: &mut Session, p: &Json) -> Result { let e = Edit::InsertCells { axis: Axis::Rows, range: RangeRef::new(r.start, CellRef::new(r.start.row, r.end.col)) }; shift_cell_features(sh, &e); let name = sh.name.clone(); - rewrite(cx, &name, &e); + rewrite(cx, &name, &e)?; let sh = cx.sheet_mut(sheet)?; for (i, c) in (r.start.col..=r.end.col).enumerate() { sh.set_value(CellRef::new(r.start.row, c), gridcraft_core::Value::text(format!("Column{}", i + 1))); diff --git a/crates/engine/src/cmd/inspect.rs b/crates/engine/src/cmd/inspect.rs index 6e2ec3e49..e91a269fb 100644 --- a/crates/engine/src/cmd/inspect.rs +++ b/crates/engine/src/cmd/inspect.rs @@ -9,7 +9,7 @@ pub fn specs() -> Vec { vec![ cmd!(query "document.inspect", "Inspect Workbook", [], None, "{} → sheets, selection, names, tables, charts, dirty, trust (local|fromFile), history", has_doc, inspect), cmd!(query "sheet.read", "Read Range", [], None, "{range?: \"A1:D10\" (default: used range), sheet?, formulas?: bool, formatted?: bool} → rows of values", has_doc, read), - cmd!(query "cell.get", "Get Cell", [], None, "{cell?, sheet?} → value, formula, display text, style, comment, link", has_doc, cell_get), + cmd!(query "cell.get", "Get Cell", [], None, "{cell?, sheet?} → value, formula, display text, style, comment, link, python {code, output, typeName, preview, stale}", has_doc, cell_get), cmd!(query "selection.stats", "Selection Statistics", [], None, "{} → count, sum, average, min, max", has_doc, sel_stats), cmd!(query "history.list", "Undo History", [], None, "{} → undo/redo labels, grouping (an undo group is open)", has_doc, history), cmd!(query "app.commands", "List Commands", [], None, "{search?} → ids, labels, params", always, list_commands), @@ -137,6 +137,13 @@ fn cell_get(s: &mut Session, p: &Json) -> Result { "merge": sh.merge_at(c).map(|m| m.a1()), "comment": sh.comments.get(&c), "hyperlink": sh.hyperlinks.get(&c), + "python": cell.and_then(|x| x.py.as_deref()).map(|py| json!({ + "code": py.code, + "output": if py.object { "object" } else { "value" }, + "typeName": py.output.as_ref().map(|o| o.type_name.as_str()), + "preview": py.output.as_ref().map(|o| o.preview.as_str()), + "stale": py.stale, + })), })) } diff --git a/crates/engine/src/cmd/pivot/create.rs b/crates/engine/src/cmd/pivot/create.rs index 5a65609c3..ce5f97257 100644 --- a/crates/engine/src/cmd/pivot/create.rs +++ b/crates/engine/src/cmd/pivot/create.rs @@ -255,7 +255,10 @@ pub(super) fn show_details(s: &mut Session, p: &Json) -> Result { } } for (c, h) in src.headers.iter().enumerate() { - sh.set_cell(CellRef::new(0, c as u32), Cell { value: gridcraft_core::Value::text(h.as_str()), formula: None, style: StyleId::default() }); + sh.set_cell( + CellRef::new(0, c as u32), + Cell { value: gridcraft_core::Value::text(h.as_str()), formula: None, style: StyleId::default(), py: None }, + ); } for (r, i) in records.iter().enumerate() { let Some(rec) = src.rows.get(*i) else { continue }; @@ -264,7 +267,7 @@ pub(super) fn show_details(s: &mut Session, p: &Json) -> Result { if v.is_empty() && style == StyleId::default() { continue; } - sh.set_cell(CellRef::new(r as u32 + 1, c as u32), Cell { value: v.clone(), formula: None, style }); + sh.set_cell(CellRef::new(r as u32 + 1, c as u32), Cell { value: v.clone(), formula: None, style, py: None }); } } let name = sh.name.clone(); diff --git a/crates/engine/src/cmd/python.rs b/crates/engine/src/cmd/python.rs new file mode 100644 index 000000000..8e7007ea1 --- /dev/null +++ b/crates/engine/src/cmd/python.rs @@ -0,0 +1,96 @@ +//! Python in cells: making and editing Python cells. This build stores the code and the last +//! result a file cached; it doesn't run Python, so a cell with new or changed code shows +//! `#BLOCKED!`. + +use std::sync::Arc; + +use gridcraft_core::{CellRef, Value}; +use gridcraft_model::{Cell, PyCode, python::MAX_FORMULA_CHARS}; +use serde_json::{Value as Json, json}; + +use super::{CommandSpec, bad, cell_param_on, cmd, edit, has_doc, str_param, target_sheet}; +use crate::{EngineError, Result, Session}; + +pub fn specs() -> Vec { + vec![cmd!( + "python.set", + "Set Python Code", + [], + None, + "{cell?, code, output?: object|value} makes the cell (default: the active cell) a Python cell with this code; the code must fit in Excel's 8,192-character formula once written as PY(…). `output` is the return type: `value` (an Excel value, the default) or `object` (a Python object)", + has_doc, + set + )] +} + +fn target_cell(s: &Session, p: &Json) -> Result<(usize, CellRef)> { + let sheet = target_sheet(s, p)?; + let at = match cell_param_on(s, p, "cell", sheet)? { + Some(c) => c, + None => s.doc()?.selection.active, + }; + Ok((sheet, at)) +} + +fn existing(s: &Session, sheet: usize, at: CellRef) -> Result> { + Ok(s.doc()?.wb.sheet(sheet).and_then(|sh| sh.cell(at)).and_then(|c| c.py.as_deref()).cloned()) +} + +fn output_mode(p: &Json, key: &str) -> Result> { + match str_param(p, key) { + None => Ok(None), + Some(m) if m.eq_ignore_ascii_case("object") => Ok(Some(true)), + Some(m) if m.eq_ignore_ascii_case("value") => Ok(Some(false)), + Some(m) => Err(bad("python.set", format!("`{key}` is `object` or `value`, not `{m}`"))), + } +} + +fn set(s: &mut Session, p: &Json) -> Result { + let (sheet, at) = target_cell(s, p)?; + let code = match p.get("code") { + Some(Json::String(c)) => c.as_str(), + Some(_) => return Err(bad("python.set", "`code` must be text")), + None => return Err(bad("python.set", "missing `code`")), + }; + let mode = output_mode(p, "output")?; + let old = existing(s, sheet, at)?; + let object = mode.or(old.as_ref().map(|o| o.object)).unwrap_or(false); + let mut py = PyCode { object, ..PyCode::new(code) }; + if !py.fits() { + return Err(bad( + "python.set", + format!("the code doesn't fit in one cell: written as PY(…) it would be longer than Excel's {MAX_FORMULA_CHARS}-character formula limit"), + )); + } + // A table header is the column's name, never a formula or code (as in Excel). + let header = + s.doc()?.wb.sheet(sheet).is_some_and(|sh| sh.tables.iter().any(|t| t.header_row && t.range.start.row == at.row && t.range.contains(at))); + if header { + return Err(EngineError::Other(format!("{} is a table header; Python cells can't go in a table's header row", at.a1()))); + } + // Unchanged code with the same return type keeps its result (and its value); new code, or a + // new return type, has none until it runs: a result of the other type would show a value and + // preview that don't belong to the cell. + let keep = old.filter(|o| o.code == py.code && o.object == py.object); + let value = match &keep { + Some(o) => { + py.output = o.output.clone(); + py.stale = o.stale; + None + } + None => Some(gridcraft_model::python::BLOCKED), + }; + edit(s, |cx| { + let sh = cx.sheet_mut(sheet)?; + let style = sh.style_id(at); + let value = value.unwrap_or_else(|| sh.cell(at).map(|c| c.value.clone()).unwrap_or(Value::Empty)); + // A spill the cell had (a formula's, or the old result's) is left to the recalculation, + // which removes it and recalculates what read it. + sh.set_cell(at, Cell { value, formula: None, style, py: Some(Arc::new(py)) }); + cx.touch(sheet, at); + Ok(()) + })?; + let d = s.doc()?; + let shown = d.wb.sheet(sheet).map(|sh| sh.value(at)).unwrap_or_default(); + Ok(json!({"cell": at.a1(), "value": crate::cmd::inspect::value_json(&shown)})) +} diff --git a/crates/engine/src/cmd/sheet.rs b/crates/engine/src/cmd/sheet.rs index fa9c915c7..69e04dae6 100644 --- a/crates/engine/src/cmd/sheet.rs +++ b/crates/engine/src/cmd/sheet.rs @@ -217,10 +217,10 @@ fn shift_sheet_features(sh: &mut Sheet, axis: Axis, at: u32, count: u32, insert: /// Rewrites every formula reference (chart series included), sparkline source and PivotTable /// source for a structural edit on `target`. -pub(super) fn rewrite(cx: &mut Ctx, target: &str, e: &Edit) { +pub(super) fn rewrite(cx: &mut Ctx, target: &str, e: &Edit) -> Result<()> { cx.edits.push((target.to_string(), e.clone())); let wb = &mut cx.wb; - super::edit::rewrite_all_formulas(wb, target, e); + super::edit::rewrite_all_formulas(wb, target, e)?; for si in 0..wb.sheets.len() { let host = wb.sheets.get(si).map(|s| s.name.clone()).unwrap_or_default(); let Some(sh) = wb.sheet_mut(si) else { continue }; @@ -246,6 +246,7 @@ pub(super) fn rewrite(cx: &mut Ctx, target: &str, e: &Edit) { } } } + Ok(()) } /// Inserting or deleting cells moves the copied block away from the stored source range, so, like @@ -303,7 +304,7 @@ fn insert_lines(s: &mut Session, p: &Json, axis: Axis) -> Result { // New lines take the format of the line before (Excel's "Format Same As Above"). inherit_line_format(sh, axis, at, count); } - rewrite(cx, &name, &Edit::Insert { axis, at, count }); + rewrite(cx, &name, &Edit::Insert { axis, at, count })?; cx.structural = true; Ok(Json::Null) })?; @@ -368,7 +369,7 @@ fn delete_lines(s: &mut Session, p: &Json, axis: Axis) -> Result { } shift_sheet_features(sh, axis, at, count, false); } - rewrite(cx, &name, &Edit::Delete { axis, at, count }); + rewrite(cx, &name, &Edit::Delete { axis, at, count })?; } cx.structural = true; Ok(Json::Null) @@ -420,7 +421,7 @@ fn move_columns(s: &mut Session, p: &Json) -> Result { sh.cells.shift_cols(before, count as i64); shift_sheet_features(sh, Axis::Cols, before, count, true); } - rewrite(cx, &name, &Edit::Insert { axis: Axis::Cols, at: before, count }); + rewrite(cx, &name, &Edit::Insert { axis: Axis::Cols, at: before, count })?; let mut touched = Vec::new(); { @@ -458,14 +459,14 @@ fn move_columns(s: &mut Session, p: &Json) -> Result { for cell in touched { cx.touch(sheet, cell); } - rewrite(cx, &name, &Edit::Move { from: source, to_row: 0, to_col: destination }); + rewrite(cx, &name, &Edit::Move { from: source, to_row: 0, to_col: destination })?; { let sh = cx.sheet_mut(sheet)?; sh.cells.shift_cols(delete_at + count, -(count as i64)); shift_sheet_features(sh, Axis::Cols, delete_at, count, false); } - rewrite(cx, &name, &Edit::Delete { axis: Axis::Cols, at: delete_at, count }); + rewrite(cx, &name, &Edit::Delete { axis: Axis::Cols, at: delete_at, count })?; *cx.sel = crate::Selection { active: CellRef::new(active_row, final_start), anchor: CellRef::new(active_row, final_start), @@ -496,7 +497,7 @@ fn insert_cells(s: &mut Session, p: &Json) -> Result { } else { RangeRef::new(CellRef::new(r.start.row.max(MAX_ROWS.saturating_sub(r.height())), r.start.col), CellRef::new(MAX_ROWS - 1, r.end.col)) }; - if sh.cells.iter_range(doomed).any(|(_, x)| x.formula.is_some() || !x.value.is_empty()) { + if sh.cells.iter_range(doomed).any(|(_, x)| x.has_content()) { return Err(EngineError::Other("To prevent possible loss of data, GridCraft cannot shift nonblank cells off of the worksheet.".into())); } } @@ -510,7 +511,7 @@ fn insert_cells(s: &mut Session, p: &Json) -> Result { } shift_cell_features(sh, &e); let name = cx.wb.sheet(sheet).map(|s| s.name.clone()).unwrap_or_default(); - rewrite(cx, &name, &e); + rewrite(cx, &name, &e)?; cx.structural = true; Ok(Json::Null) })?; @@ -545,7 +546,7 @@ fn delete_cells(s: &mut Session, p: &Json) -> Result { shift_cell_features(sh, &e); } // References into the deleted cells become #REF!; the rest move. - rewrite(cx, &name, &e); + rewrite(cx, &name, &e)?; cx.structural = true; Ok(Json::Null) })?; diff --git a/crates/engine/src/cmd/sheet/tabs.rs b/crates/engine/src/cmd/sheet/tabs.rs index 176c1d542..ced855297 100644 --- a/crates/engine/src/cmd/sheet/tabs.rs +++ b/crates/engine/src/cmd/sheet/tabs.rs @@ -63,6 +63,7 @@ pub(super) fn delete_sheet(s: &mut Session, p: &Json) -> Result { } } } + crate::cmd::edit::rewrite_python_refs(&mut cx.wb, &|_, e| gridcraft_formula::adjust::delete_sheet(e, &name, &order))?; // Chart series into the deleted sheet become #REF!, like formulas. crate::cmd::edit::rewrite_cell_links(&mut cx.wb, |_, e| gridcraft_formula::adjust::delete_sheet(e, &name, &order)); cx.wb.names.retain(|n| n.scope != Some(i)); @@ -133,6 +134,7 @@ pub(super) fn rename_sheet(s: &mut Session, p: &Json) -> Result { n.formula = gridcraft_formula::print(&gridcraft_formula::adjust::rename_sheet(e, &old, &name)); } } + crate::cmd::edit::rewrite_python_refs(&mut cx.wb, &|_, e| gridcraft_formula::adjust::rename_sheet(e, &old, &name))?; cx.structural = true; Ok(Json::Null) }) diff --git a/crates/engine/src/cmd/solver.rs b/crates/engine/src/cmd/solver.rs index 12c182799..c2d4b28c9 100644 --- a/crates/engine/src/cmd/solver.rs +++ b/crates/engine/src/cmd/solver.rs @@ -303,7 +303,7 @@ fn run(s: &mut Session, p: &Json) -> Result { let sh = cx.sheet_mut(sheet)?; for (c, v) in model.vars.iter().zip(x) { let style = sh.style_id(*c); - sh.set_cell(*c, Cell { value: Value::number(*v), formula: None, style }); + sh.set_cell(*c, Cell { value: Value::number(*v), formula: None, style, py: None }); } for c in &model.vars { cx.touch(sheet + count, *c); diff --git a/crates/engine/src/cmd/solver/model.rs b/crates/engine/src/cmd/solver/model.rs index 11c44546b..c7ccd6c96 100644 --- a/crates/engine/src/cmd/solver/model.rs +++ b/crates/engine/src/cmd/solver/model.rs @@ -263,7 +263,8 @@ pub(super) fn parse_model(s: &Session, p: &Json) -> Result { } let mut seen = std::collections::HashSet::new(); vars.retain(|c| seen.insert(*c)); - if vars.iter().any(|c| sh.cell(*c).is_some_and(|c| c.formula.is_some())) { + // Python cells count as formulas: a solution written over one would lose its code. + if vars.iter().any(|c| sh.cell(*c).is_some_and(|c| c.formula.is_some() || c.py.is_some())) { return Err(EngineError::Other("Variable cells must hold constants, not formulas.".into())); } let mut cons = Vec::new(); diff --git a/crates/engine/src/cmd/solver/model_range.rs b/crates/engine/src/cmd/solver/model_range.rs index 7d47b7176..b68b8bfb9 100644 --- a/crates/engine/src/cmd/solver/model_range.rs +++ b/crates/engine/src/cmd/solver/model_range.rs @@ -107,12 +107,12 @@ fn check_column_spills(sheet: &Sheet, column: RangeRef, ours: &[CellRef]) -> Res Ok(()) } -/// Refused, unless `overwrite`, when a cell of the model's column holds a value or a formula -/// (a model saved there before included): Save Model then asks before it replaces data, as -/// Excel does. The message carries [`OVERWRITE_ERROR`], which the dialog turns into its +/// Refused, unless `overwrite`, when a cell of the model's column holds a value, a formula or +/// Python code (a model saved there before included): Save Model then asks before it replaces +/// data, as Excel does. The message carries [`OVERWRITE_ERROR`], which the dialog turns into its /// question. fn check_column_empty(sheet: &Sheet, column: RangeRef, overwrite: bool) -> Result<()> { - let taken = column.iter().find(|c| sheet.cell(*c).is_some_and(|x| !x.value.is_empty() || x.formula.is_some())); + let taken = column.iter().find(|c| sheet.cell(*c).is_some_and(Cell::has_content)); match taken { Some(c) if !overwrite => Err(EngineError::Other(format!( "The range {} {OVERWRITE_ERROR} ({} isn't empty); pass overwrite: true to replace it.", @@ -176,7 +176,7 @@ pub(super) fn save_model_cells(s: &mut Session, p: &Json) -> Result { } for c in &stale { let style = sh.style_id(*c); - sh.set_cell(*c, Cell { value: Value::Empty, formula: None, style }); + sh.set_cell(*c, Cell { value: Value::Empty, formula: None, style, py: None }); } for c in cells.into_iter().chain(stale) { cx.touch(sheet, c); diff --git a/crates/engine/src/cmd/solver/reports.rs b/crates/engine/src/cmd/solver/reports.rs index 78800cf41..19500cab0 100644 --- a/crates/engine/src/cmd/solver/reports.rs +++ b/crates/engine/src/cmd/solver/reports.rs @@ -352,7 +352,7 @@ pub(super) fn write_reports(cx: &mut Ctx, at: usize, built: Vec<(String, Lines)> let mut sheet = Sheet::new(name); for (r, c, v, b) in lines { let style = if b { bold } else { gridcraft_model::StyleId::DEFAULT }; - sheet.set_cell(CellRef::new(r, c), Cell { value: v, formula: None, style }); + sheet.set_cell(CellRef::new(r, c), Cell { value: v, formula: None, style, py: None }); } for (c, w) in [(0u32, 14.0f32), (1, 110.0), (2, 140.0), (3, 120.0), (4, 120.0), (5, 155.0), (6, 140.0), (7, 140.0)] { sheet.cols.insert(c, gridcraft_model::LineInfo { size: Some(w), ..Default::default() }); diff --git a/crates/engine/src/cmd/whatif.rs b/crates/engine/src/cmd/whatif.rs index c801c7359..1a0866e83 100644 --- a/crates/engine/src/cmd/whatif.rs +++ b/crates/engine/src/cmd/whatif.rs @@ -213,7 +213,7 @@ impl Trial { for (si, c, v) in sets { if let Some(sh) = wb.sheet_mut(*si) { let style = sh.style_id(*c); - sh.set_cell(*c, Cell { value: v.clone(), formula: None, style }); + sh.set_cell(*c, Cell { value: v.clone(), formula: None, style, py: None }); keys.push((*si, *c)); } } @@ -329,7 +329,8 @@ fn goal_seek(s: &mut Session, p: &Json) -> Result { if !has_formula(ssi, set) { return Err(EngineError::Other("Cell must contain a formula.".into())); } - if has_formula(csi, chg) { + // A Python cell is a formula to Excel (`=PY(…)`): writing a number over it would lose its code. + if has_formula(csi, chg) || d.wb.sheet(csi).and_then(|sh| sh.cell(chg)).is_some_and(|c| c.py.is_some()) { return Err(EngineError::Other("Cell must contain a value.".into())); } let x0 = d.wb.sheet(csi).map(|sh| sh.value(chg)).and_then(|v| v.as_f64()).unwrap_or(0.0); @@ -342,7 +343,7 @@ fn goal_seek(s: &mut Session, p: &Json) -> Result { edit(s, |cx| { let sh = cx.sheet_mut(csi)?; let style = sh.style_id(chg); - sh.set_cell(chg, Cell { value: Value::number(r.x), formula: None, style }); + sh.set_cell(chg, Cell { value: Value::number(r.x), formula: None, style, py: None }); cx.touch(csi, chg); Ok(()) })?; @@ -422,7 +423,7 @@ fn data_table(s: &mut Session, p: &Json) -> Result { for (c, v) in out { let sh = cx.sheet_mut(si)?; let style = sh.style_id(c); - sh.set_cell(c, Cell { value: v, formula: None, style }); + sh.set_cell(c, Cell { value: v, formula: None, style, py: None }); cx.touch(si, c); } Ok(()) diff --git a/crates/engine/src/cmd/whatif/advanced_filter.rs b/crates/engine/src/cmd/whatif/advanced_filter.rs index e21004342..52630a3ef 100644 --- a/crates/engine/src/cmd/whatif/advanced_filter.rs +++ b/crates/engine/src/cmd/whatif/advanced_filter.rs @@ -186,7 +186,7 @@ pub(super) fn advanced_filter(s: &mut Session, p: &Json) -> Result { // The full output footprint was checked above. let to = CellRef::new(dest.row + i as u32, dest.col + c - list.start.col); let style = sh.style_id(src); - (to, Cell { value: sh.value(src), formula: None, style }) + (to, Cell { value: sh.value(src), formula: None, style, py: None }) }) }) .collect(); diff --git a/crates/engine/src/cmd/whatif/consolidate.rs b/crates/engine/src/cmd/whatif/consolidate.rs index ec5fbb54a..149b72bc7 100644 --- a/crates/engine/src/cmd/whatif/consolidate.rs +++ b/crates/engine/src/cmd/whatif/consolidate.rs @@ -185,7 +185,7 @@ pub(super) fn consolidate(s: &mut Session, p: &Json) -> Result { if v.is_empty() && style == StyleId::DEFAULT { sh.remove_cell(c); } else { - sh.set_cell(c, Cell { value: v, formula: None, style }); + sh.set_cell(c, Cell { value: v, formula: None, style, py: None }); } cx.touch(dsi, c); } diff --git a/crates/engine/src/cmd/whatif/scenarios.rs b/crates/engine/src/cmd/whatif/scenarios.rs index fc442f8f9..75bf147a0 100644 --- a/crates/engine/src/cmd/whatif/scenarios.rs +++ b/crates/engine/src/cmd/whatif/scenarios.rs @@ -160,7 +160,7 @@ pub(super) fn scenario_show(s: &mut Session, p: &Json) -> Result { let cell = match v.as_f64() { Some(n) => { let style = cx.wb.sheet(si).map(|sh| sh.style_id(*c)).unwrap_or_default(); - Some(Cell { value: Value::Number(n), formula: None, style }) + Some(Cell { value: Value::Number(n), formula: None, style, py: None }) } None => crate::cmd::edit::input_to_cell(&crate::cmd::edit::json_to_input(v), si, *c, &mut cx.wb)?, }; @@ -245,7 +245,7 @@ pub(super) fn scenario_summary(s: &mut Session, p: &Json) -> Result { let bold = cx.wb.styles.derive(StyleId::DEFAULT, |st| st.font.bold = true); let mut sheet = Sheet::new(name.clone()); let put = |sheet: &mut Sheet, row: u32, col: u32, v: Value, style: StyleId| { - sheet.set_cell(CellRef::new(row, col), Cell { value: v, formula: None, style }); + sheet.set_cell(CellRef::new(row, col), Cell { value: v, formula: None, style, py: None }); }; put(&mut sheet, 1, 1, Value::text("Scenario Summary"), bold); put(&mut sheet, 2, 2, Value::text("Current Values:"), bold); diff --git a/crates/engine/src/display.rs b/crates/engine/src/display.rs index aa8bdb22d..2189d6c64 100644 --- a/crates/engine/src/display.rs +++ b/crates/engine/src/display.rs @@ -90,14 +90,14 @@ pub fn edit_text(wb: &Workbook, sheet: &Sheet, c: CellRef) -> String { if reads_back(&iso) { iso } else { raw } } -/// Whether `c` has a formula that must not be shown: its cell is formatted Hidden on a -/// protected sheet. A spilled cell follows its anchor. +/// Whether `c` has a formula or Python code that must not be shown: its cell is formatted +/// Hidden on a protected sheet. A spilled cell follows its anchor. pub fn formula_hidden(wb: &Workbook, sheet: &Sheet, c: CellRef) -> bool { if !sheet.is_protected() { return false; } let at = match sheet.cell(c) { - Some(x) if x.formula.is_some() => c, + Some(x) if x.formula.is_some() || x.py.is_some() => c, _ => match sheet.spill_ranges.iter().find(|(_, r)| r.contains(c)) { Some((anchor, _)) => *anchor, None => return false, diff --git a/crates/engine/src/fill.rs b/crates/engine/src/fill.rs index 1201721da..c63d7c983 100644 --- a/crates/engine/src/fill.rs +++ b/crates/engine/src/fill.rs @@ -171,10 +171,12 @@ pub fn fill(cx: &mut Ctx, sheet: usize, src: RangeRef, target: RangeRef, mode: F cell.formula = Some(Arc::new(Formula::from_expr(shifted))); cell.value = Value::Empty; } - } else if let Some(v) = match per_cell.get(src_i) { - Some(p) => p.value(k as i64 / src_cells.len() as i64 + 1, forward, 1), - None => pattern.value(step, forward, src_cells.len()), - } { + } else if cell.py.is_none() + && let Some(v) = match per_cell.get(src_i) { + Some(p) => p.value(k as i64 / src_cells.len() as i64 + 1, forward, 1), + None => pattern.value(step, forward, src_cells.len()), + } + { cell.value = v; } cell.style = style; @@ -296,7 +298,8 @@ impl Pattern { fn detect(cells: &[(CellRef, Option)], custom: &[Vec], sys: gridcraft_core::DateSystem, fmt: &str) -> Pattern { let vals: Vec = cells.iter().map(|(_, c)| c.as_ref().map(|c| c.value.clone()).unwrap_or_default()).collect(); - if cells.iter().any(|(_, c)| c.as_ref().is_some_and(|c| c.formula.is_some())) { + // Formulas and Python cells copy. + if cells.iter().any(|(_, c)| c.as_ref().is_some_and(|c| c.formula.is_some() || c.py.is_some())) { return Pattern::Copy; } // Numbers. @@ -598,7 +601,7 @@ pub fn series(cx: &mut Ctx, sheet: usize, options: SeriesOptions<'_>) -> Result< break; } let c = if rows { CellRef::new(lane, r.start.col + i) } else { CellRef::new(r.start.row + i, lane) }; - writes.push((c, Cell { value: Value::number(round15(v)), formula: None, style })); + writes.push((c, Cell { value: Value::number(round15(v)), formula: None, style, py: None })); } } let sh = cx.sheet_mut(sheet)?; @@ -657,7 +660,7 @@ pub fn flash_fill(cx: &mut Ctx, sheet: usize, at: CellRef) -> Result { } let sh = cx.sheet_mut(sheet)?; for (c, out) in &writes { - sh.set_cell(*c, Cell { value: Value::text(out.as_str()), formula: None, style }); + sh.set_cell(*c, Cell { value: Value::text(out.as_str()), formula: None, style, py: None }); } for (c, _) in writes { cx.touch(sheet, c); diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index 69b1ebfa7..ce1fb40b0 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -118,13 +118,142 @@ fn decrypt_message(name: &str, e: gridcraft_xlsx::DecryptError) -> String { } } -/// Parses bytes that aren't encrypted, then takes the workbook's macros out of their hidden names. +/// Parses bytes that aren't encrypted, then takes the workbook's macros out of their hidden names +/// and adopts the Python cells they hold (see [`adopt_python`]) within the limits of +/// [`enforce_python_limits`]. fn open_plain(name: &str, bytes: &[u8], csv_as_text: bool) -> Result<(Workbook, Vec)> { let (mut wb, mut warnings) = read_plain(name, bytes, csv_as_text)?; warnings.extend(crate::macros::load(&mut wb)); + warnings.extend(adopt_python(&mut wb)); + warnings.extend(enforce_python_limits(&mut wb)); Ok((wb, warnings)) } +/// The most result cells [`adopt_python`] rebuilds from a file's cached spills, over all its +/// Python cells (each one is also limited to [`gridcraft_model::python::MAX_RESULT_CELLS`]). +pub const MAX_ADOPTED_RESULT_CELLS: u64 = 4_000_000; + +/// Python cells from a file: a cell whose formula is `PY("…",0|1)` (a format that kept it as a +/// formula) becomes a Python cell, and a Python cell whose spilled values the file cached gets +/// them back as its result, so it shows them until it runs again. Results are rebuilt for at most +/// [`MAX_ADOPTED_RESULT_CELLS`] cells in all; the rest keep only their own cached value, and the +/// values they spilled stay as plain values. Returns the warnings. +pub fn adopt_python(wb: &mut Workbook) -> Vec { + let mut budget = MAX_ADOPTED_RESULT_CELLS; + let mut skipped = 0usize; + for si in 0..wb.sheets.len() { + let Some(sh) = wb.sheets.get(si) else { continue }; + let formulas: Vec<(CellRef, gridcraft_model::PyCode)> = sh + .cells + .iter() + .filter_map(|(c, cell)| cell.formula.as_ref().and_then(|f| gridcraft_model::PyCode::from_formula(&f.text)).map(|p| (c, p))) + .collect(); + let max = gridcraft_model::python::MAX_RESULT_CELLS as u64; + let spills: Vec<(CellRef, RangeRef)> = sh + .spill_ranges + .iter() + .filter(|(a, r)| r.count() <= max && sh.cell(**a).and_then(|c| c.py.as_ref()).is_some_and(|p| p.output.is_none())) + .map(|(a, r)| (*a, *r)) + .collect(); + if formulas.is_empty() && spills.is_empty() { + continue; + } + let Some(sh) = wb.sheet_mut(si) else { continue }; + for (c, py) in formulas { + if let Some(cell) = sh.cells.get_mut(c) { + cell.formula = None; + cell.py = Some(std::sync::Arc::new(py)); + } + } + for (anchor, range) in spills { + let Some(left) = budget.checked_sub(range.count()) else { + skipped += 1; + // The cell keeps only its own value; the values it spilled stay as plain values, so + // nothing shows as a spill that no result holds, and nothing the file showed is lost. + sh.settle_spill(anchor); + continue; + }; + budget = left; + let data: Vec = + range.iter().map(|c| if c == anchor { sh.cell(c).map(|x| x.value.clone()).unwrap_or_default() } else { sh.value(c) }).collect(); + let Some(array) = gridcraft_core::Array::new(range.height() as usize, range.width() as usize, data) else { continue }; + if let Some(py) = sh.cells.get_mut(anchor).and_then(|x| x.py.as_mut()) { + std::sync::Arc::make_mut(py).output = + Some(gridcraft_model::PyOutput { value: array.into(), type_name: "list".into(), preview: String::new(), convertible: true }); + } + } + } + if skipped == 0 { + return Vec::new(); + } + vec![format!( + "The file's Python results are larger than {MAX_ADOPTED_RESULT_CELLS} cells in all, so {skipped} Python cell(s) have no result until they run again; the values they spilled were kept as plain values." + )] +} + +/// A file is somebody else's work: a Python cell whose code doesn't fit in Excel's +/// 8,192-character formula keeps only its cached value, and a kept result that is not a +/// well-formed array of at most [`gridcraft_model::python::MAX_RESULT_CELLS`] cells is dropped +/// (the cell keeps its value). Either way the values the cell spilled stay as plain values +/// ([`gridcraft_model::Sheet::settle_spill`]). Returns the warnings. +pub fn enforce_python_limits(wb: &mut Workbook) -> Vec { + let mut long = Vec::new(); + for si in 0..wb.sheets.len() { + let Some(sh) = wb.sheets.get(si) else { continue }; + let mut over = Vec::new(); + let mut bad_output = Vec::new(); + for (c, cell) in sh.cells.iter() { + let Some(py) = &cell.py else { continue }; + if !py.fits() { + long.push(format!("{}!{}", sh.name, c.a1())); + over.push(c); + } else if py.output.as_ref().is_some_and(|o| !sane_python_output(&o.value)) { + bad_output.push(c); + } + } + if over.is_empty() && bad_output.is_empty() { + continue; + } + let dropped: Vec = over.iter().chain(&bad_output).copied().collect(); + let Some(sh) = wb.sheet_mut(si) else { continue }; + for c in over { + if let Some(cell) = sh.cells.get_mut(c) { + cell.py = None; + } + } + for c in bad_output { + if let Some(py) = sh.cells.get_mut(c).and_then(|x| x.py.as_mut()) { + std::sync::Arc::make_mut(py).output = None; + } + } + // A dropped result (or code dropped to its value) leaves the values it spilled as plain + // values, as the file showed them, with no live spill. + for c in dropped { + sh.settle_spill(c); + } + } + let Some(first) = long.first() else { return vec![] }; + let more = if long.len() > 1 { format!(" (and {} more)", long.len() - 1) } else { String::new() }; + vec![format!( + "A Python cell's code is longer than Excel's {}-character formula limit, so {first}{more} kept only its last calculated value.", + gridcraft_model::python::MAX_FORMULA_CHARS + )] +} + +/// A Python result a file may hold: a scalar, or an array whose size matches its data and has at +/// most [`gridcraft_model::python::MAX_RESULT_CELLS`] cells. +fn sane_python_output(v: &gridcraft_core::Value) -> bool { + match v { + gridcraft_core::Value::Array(a) => { + a.rows >= 1 + && a.cols >= 1 + && a.rows.checked_mul(a.cols).is_some_and(|n| n == a.data.len() && n <= gridcraft_model::python::MAX_RESULT_CELLS) + } + _ => true, + } +} + +/// Parses bytes that aren't encrypted, as they are. fn read_plain(name: &str, bytes: &[u8], csv_as_text: bool) -> Result<(Workbook, Vec)> { let kind = FileKind::from_path(name); let sniffed = gridcraft_xlsx::sniff(bytes); diff --git a/crates/engine/src/io/tests.rs b/crates/engine/src/io/tests.rs index e4841b562..9051c5c09 100644 --- a/crates/engine/src/io/tests.rs +++ b/crates/engine/src/io/tests.rs @@ -461,17 +461,17 @@ fn csv_writes_displayed_text() { (styled("yyyy-mm-dd"), styled("$#,##0.00"), styled("0.0%"), styled("# ?/?"), styled("@"), styled("0.00"), styled("#,##0")); let sh = wb.sheet_mut(0).unwrap(); let cells = [ - ("A1", Cell { value: Value::Number(46023.0), formula: None, style: date }), - ("B1", Cell { value: Value::Number(1234.5), formula: None, style: money }), - ("C1", Cell { value: Value::Number(0.125), formula: None, style: pct }), - ("D1", Cell { value: Value::Number(1.5), formula: None, style: frac }), - ("E1", Cell { value: Value::text("007"), formula: None, style: text }), - ("F1", Cell { value: Value::Error(CellError::NA), formula: None, style: money }), - ("G1", Cell { value: Value::Bool(true), formula: None, style: StyleId::DEFAULT }), + ("A1", Cell { value: Value::Number(46023.0), formula: None, style: date, py: None }), + ("B1", Cell { value: Value::Number(1234.5), formula: None, style: money, py: None }), + ("C1", Cell { value: Value::Number(0.125), formula: None, style: pct, py: None }), + ("D1", Cell { value: Value::Number(1.5), formula: None, style: frac, py: None }), + ("E1", Cell { value: Value::text("007"), formula: None, style: text, py: None }), + ("F1", Cell { value: Value::Error(CellError::NA), formula: None, style: money, py: None }), + ("G1", Cell { value: Value::Bool(true), formula: None, style: StyleId::DEFAULT, py: None }), ("A2", Cell { value: Value::Number(46024.0), style: date, ..Cell::formula(Formula::new("A1+1")) }), - ("B2", Cell { value: Value::Number(12345678.9), formula: None, style: fixed }), - ("C2", Cell { value: Value::Number(-0.25), formula: None, style: StyleId::DEFAULT }), - ("D2", Cell { value: Value::Number(1234567.0), formula: None, style: thousands }), + ("B2", Cell { value: Value::Number(12345678.9), formula: None, style: fixed, py: None }), + ("C2", Cell { value: Value::Number(-0.25), formula: None, style: StyleId::DEFAULT, py: None }), + ("D2", Cell { value: Value::Number(1234567.0), formula: None, style: thousands, py: None }), ]; for (a, c) in cells { sh.set_cell(CellRef::parse(a).unwrap(), c); diff --git a/crates/engine/src/macros/resolve.rs b/crates/engine/src/macros/resolve.rs index 3825dcc96..c5c2da8d7 100644 --- a/crates/engine/src/macros/resolve.rs +++ b/crates/engine/src/macros/resolve.rs @@ -614,7 +614,10 @@ mod tests { let mut wb = wb(); let sh = wb.sheet_mut(0).unwrap(); let formula = Some(std::sync::Arc::new(gridcraft_model::Formula::new("\"Amo\"&\"unt\""))); - sh.set_cell(CellRef::new(1, 2), gridcraft_model::Cell { value: gridcraft_core::Value::text("Amount"), formula, style: Default::default() }); + sh.set_cell( + CellRef::new(1, 2), + gridcraft_model::Cell { value: gridcraft_core::Value::text("Amount"), formula, style: Default::default(), py: None }, + ); sh.show_formulas = true; let inputs = vec![("Input".to_string(), place("B2:D11"), true)]; let env = Env { wb: &wb, active_sheet: 0, inputs: &inputs, output: None, item: None }; diff --git a/crates/engine/src/macros/run.rs b/crates/engine/src/macros/run.rs index fcb6768f3..71c23aabe 100644 --- a/crates/engine/src/macros/run.rs +++ b/crates/engine/src/macros/run.rs @@ -1237,26 +1237,39 @@ pub(crate) fn pair_sheets(before: &Workbook, after: &Workbook) -> (Vec<(usize, O (pairs, spare.collect()) } -/// Only a formula's cached value changed (a recalculation, not a write). +/// Only a formula's (or a Python cell's) cached value changed (a recalculation, not a write): a +/// Python cell whose spill a step blocked shows #SPILL! with its code untouched. fn recalculated_only(b: Option<&Cell>, a: Option<&Cell>) -> bool { match (b, a) { (Some(b), Some(a)) => match (&b.formula, &a.formula) { (Some(x), Some(y)) => x.text == y.text && x.array == y.array && b.style == a.style, + (None, None) => match (&b.py, &a.py) { + (Some(x), Some(y)) => x.code == y.code && x.object == y.object && b.style == a.style, + _ => false, + }, _ => false, }, _ => false, } } +/// Python code is content, as a formula is, whatever the cell shows. fn has_content(c: &Cell) -> bool { - c.formula.is_some() || !(c.value.is_empty() || c.value.as_text() == Some("")) + c.formula.is_some() || c.py.is_some() || !(c.value.is_empty() || c.value.as_text() == Some("")) } +/// Whether a write changed what the person entered: the formula, the Python code and its return +/// type, or the value. A Python cell replaced by its own cached value, or by other code that +/// shows the same `#BLOCKED!`, has changed. fn content_differs(b: Option<&Cell>, a: Option<&Cell>) -> bool { match (b, a) { (Some(b), Some(a)) => match (&b.formula, &a.formula) { (Some(x), Some(y)) => x.text != y.text, - (None, None) => b.value != a.value, + (None, None) => match (&b.py, &a.py) { + (Some(x), Some(y)) => x.code != y.code || x.object != y.object, + (None, None) => b.value != a.value, + _ => true, + }, _ => true, }, (Some(b), None) => has_content(b), diff --git a/crates/engine/src/pivot.rs b/crates/engine/src/pivot.rs index fb72492d5..bfaccace4 100644 --- a/crates/engine/src/pivot.rs +++ b/crates/engine/src/pivot.rs @@ -228,7 +228,7 @@ pub fn refresh(wb: &mut Workbook, sheet: usize, idx: usize, replace: bool) -> Re } if !replace { for (c, cell) in sh.cells.iter_range(area) { - if (!cell.value.is_empty() || cell.formula.is_some()) && !pt.last_range.is_some_and(|r| r.contains(c)) { + if cell.has_content() && !pt.last_range.is_some_and(|r| r.contains(c)) { return Err(format!( "There's already data in {}!{}. Do you want to replace it? (Run again with replace: true to overwrite.)", quote_sheet(&sh.name), @@ -278,7 +278,10 @@ pub fn refresh(wb: &mut Workbook, sheet: usize, idx: usize, replace: bool) -> Re if oc.value.is_empty() && id == StyleId::default() { continue; } - cells.push((CellRef::new(origin.row + ri as u32, origin.col + ci as u32), Cell { value: oc.value.clone(), formula: None, style: id })); + cells.push(( + CellRef::new(origin.row + ri as u32, origin.col + ci as u32), + Cell { value: oc.value.clone(), formula: None, style: id, py: None }, + )); } } // Column widths that fit the labels. diff --git a/crates/engine/src/selection.rs b/crates/engine/src/selection.rs index 4df6fbe4a..0e00e4869 100644 --- a/crates/engine/src/selection.rs +++ b/crates/engine/src/selection.rs @@ -190,7 +190,7 @@ pub fn jump(sheet: &Sheet, from: CellRef, dr: i64, dc: i64) -> CellRef { /// The contiguous data region around `c` (Ctrl+A / Ctrl+Shift+8, AutoSum, sort, tables). pub fn current_region(sheet: &Sheet, c: CellRef) -> RangeRef { - let filled = |c: CellRef| sheet.cell(c).is_some_and(|x| !x.value.is_empty() || x.formula.is_some()) || sheet.spill.contains_key(&c); + let filled = |c: CellRef| sheet.cell(c).is_some_and(gridcraft_model::Cell::has_content) || sheet.spill.contains_key(&c); // Whether a row (or column) has a filled cell between two columns (rows), inclusive. let row_has = |row: u32, a: u32, b: u32| (a..=b).any(|col| filled(CellRef::new(row, col))); let col_has = |col: u32, a: u32, b: u32| (a..=b).any(|row| filled(CellRef::new(row, col))); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 5afcbf9fa..22e71330a 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5246,6 +5246,9 @@ fn a_script_cannot_switch_windows() { mod equation; mod macros; mod objects; +mod python_cells; +mod python_commands; +mod python_files; mod seams; #[test] diff --git a/crates/engine/src/tests/python_cells.rs b/crates/engine/src/tests/python_cells.rs new file mode 100644 index 000000000..63cf1d92b --- /dev/null +++ b/crates/engine/src/tests/python_cells.rs @@ -0,0 +1,532 @@ +//! Python cells under edits: what keeps, moves, rewrites or removes their code. + +use std::sync::Arc; + +use gridcraft_core::{CellRef, Value}; +use gridcraft_model::{Cell, PyCode}; +use serde_json::json; + +use super::s; +use crate::Session; + +/// Makes `cell` on sheet `sheet` a Python cell with `code` and a cached value. +fn plant(s: &mut Session, sheet: usize, cell: &str, code: &str) { + let d = s.doc_mut().unwrap(); + let mut wb = (*d.wb).clone(); + let sh = wb.sheet_mut(sheet).unwrap(); + let at = CellRef::parse(cell).unwrap(); + let style = sh.style_id(at); + sh.set_cell(at, Cell { value: Value::Number(1.0), formula: None, style, py: Some(Arc::new(PyCode::new(code))) }); + d.wb = Arc::new(wb); +} + +fn code(s: &Session, sheet: usize, cell: &str) -> Option { + let d = s.doc().unwrap(); + d.wb.sheet(sheet).unwrap().cell(CellRef::parse(cell).unwrap()).and_then(|c| c.py.as_ref()).map(|p| p.code.clone()) +} + +#[test] +fn typing_py_formulas_points_to_python_set() { + let mut s = s(); + let e = s.execute("cell.set", json!({"cell": "A1", "input": "=PY(\"1+1\",0)"})).unwrap_err(); + assert!(e.to_string().contains("python.set"), "{e}"); + assert!(s.execute("cell.set", json!({"cell": "A1", "input": "=py"})).is_err()); + // Other functions starting with PY are not affected. + s.execute("cell.set", json!({"cell": "A2", "input": "=PI()"})).unwrap(); +} + +#[test] +fn entering_or_clearing_a_python_cell_removes_its_code() { + let mut s = s(); + plant(&mut s, 0, "A1", "1 + 1"); + plant(&mut s, 0, "A2", "2 + 2"); + plant(&mut s, 0, "A3", "3 + 3"); + let got = s.execute("cell.get", json!({"cell": "A1"})).unwrap(); + assert_eq!(got["input"], "1 + 1"); + assert_eq!(got["python"]["code"], "1 + 1"); + assert_eq!(got["python"]["output"], "value"); + s.execute("cell.set", json!({"cell": "A1", "input": "hello"})).unwrap(); + assert_eq!(code(&s, 0, "A1"), None); + s.execute("edit.clearFormats", json!({"range": "A2"})).unwrap(); + assert_eq!(code(&s, 0, "A2").as_deref(), Some("2 + 2"), "clearing formats keeps the code"); + s.execute("edit.clearContents", json!({"range": "A2"})).unwrap(); + assert_eq!(code(&s, 0, "A2"), None); + s.execute("edit.clearAll", json!({"range": "A3"})).unwrap(); + assert_eq!(code(&s, 0, "A3"), None); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(code(&s, 0, "A3").as_deref(), Some("3 + 3"), "undo brings the code back"); +} + +#[test] +fn copy_fill_and_sort_carry_the_code_and_paste_values_drops_it() { + let mut s = s(); + plant(&mut s, 0, "A1", "xl('B1') * 2"); + s.execute("edit.copy", json!({"range": "A1"})).unwrap(); + s.execute("edit.paste", json!({"at": "C1"})).unwrap(); + assert_eq!(code(&s, 0, "C1").as_deref(), Some("xl('B1') * 2"), "the code is copied as it is"); + s.execute("edit.pasteSpecial", json!({"at": "D1", "what": "values"})).unwrap(); + assert_eq!(code(&s, 0, "D1"), None); + assert_eq!(super::v(&s, "D1"), Value::Number(1.0), "the cached value is pasted"); + s.execute("edit.fillDown", json!({"range": "A1:A3"})).unwrap(); + assert_eq!(code(&s, 0, "A3").as_deref(), Some("xl('B1') * 2")); + // Sorting moves Python cells with their rows. + s.execute("range.setValues", json!({"range": "E1", "values": [[3], [1], [2]]})).unwrap(); + plant(&mut s, 0, "F1", "'three'"); + plant(&mut s, 0, "F2", "'one'"); + s.execute("data.sort", json!({"range": "E1:F3", "keys": [{"column": "E"}]})).unwrap(); + assert_eq!(code(&s, 0, "F1").as_deref(), Some("'one'")); + assert_eq!(code(&s, 0, "F3").as_deref(), Some("'three'")); +} + +#[test] +fn xl_references_follow_inserts_moves_and_renames() { + let mut s = s(); + s.execute("home.insertSheet", json!({})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 0})).unwrap(); + let other = s.doc().unwrap().wb.sheet(1).unwrap().name.clone(); + plant(&mut s, 0, "C5", &format!("a = xl(\"A1:A3\")\nb = xl('{other}!B2', headers=True)\nc = xl(dynamic)\n# xl(\"A1\") stays in comments")); + s.execute("home.insertRows", json!({"rows": "2:2"})).unwrap(); + let c = code(&s, 0, "C6").unwrap(); + assert!(c.starts_with("a = xl(\"A1:A4\")\n"), "{c}"); + assert!(c.contains(&format!("xl('{other}!B2', headers=True)")), "{c}"); + assert!(c.contains("# xl(\"A1\") stays"), "{c}"); + s.execute("sheet.rename", json!({"sheet": 1, "name": "Data"})).unwrap(); + assert!(code(&s, 0, "C6").unwrap().contains("xl('Data!B2', headers=True)")); + // Cutting the cells a reference names moves the reference with them. + s.execute("edit.cut", json!({"range": "A1:A4"})).unwrap(); + s.execute("edit.paste", json!({"at": "H1"})).unwrap(); + assert!(code(&s, 0, "C6").unwrap().starts_with("a = xl(\"H1:H4\")"), "{:?}", code(&s, 0, "C6")); + // Deleting the referenced cells leaves #REF!, which fails when the code runs. + // The selection is elsewhere, so only the named column goes. + s.execute("selection.set", json!({"range": "A1"})).unwrap(); + s.execute("home.deleteColumns", json!({"cols": "H:H"})).unwrap(); + assert!(code(&s, 0, "C6").unwrap().starts_with("a = xl(\"#REF!\")"), "{:?}", code(&s, 0, "C6")); +} + +#[test] +fn a_rename_to_a_name_that_needs_quotes_rewrites_single_quoted_references() { + let mut s = s(); + s.execute("home.insertSheet", json!({})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 0})).unwrap(); + let other = s.doc().unwrap().wb.sheet(1).unwrap().name.clone(); + plant(&mut s, 0, "A1", &format!("a = xl('{other}!B2')\nb = xl(\"{other}!C3\")")); + s.execute("sheet.rename", json!({"sheet": 1, "name": "Q4 2026"})).unwrap(); + assert_eq!(code(&s, 0, "A1").unwrap(), "a = xl(\"'Q4 2026'!B2\")\nb = xl(\"'Q4 2026'!C3\")"); + // A name with both quotes in it: the literal's own quote is escaped. + s.execute("sheet.rename", json!({"sheet": 1, "name": "Bob's \"Q4\""})).unwrap(); + let c = code(&s, 0, "A1").unwrap(); + assert_eq!(c, "a = xl(\"'Bob''s \\\"Q4\\\"'!B2\")\nb = xl(\"'Bob''s \\\"Q4\\\"'!C3\")"); + let refs: Vec> = gridcraft_model::python::xl_calls(&c).into_iter().map(|x| x.reference).collect(); + assert_eq!(refs, [Some("'Bob''s \"Q4\"'!B2".to_string()), Some("'Bob''s \"Q4\"'!C3".to_string())]); + // And back to a plain name. + s.execute("sheet.rename", json!({"sheet": 1, "name": "Data"})).unwrap(); + assert_eq!(code(&s, 0, "A1").unwrap(), "a = xl(\"Data!B2\")\nb = xl(\"Data!C3\")"); +} + +#[test] +fn a_cut_to_another_sheet_keeps_references_on_the_source_sheet() { + let mut s = s(); + s.execute("home.insertSheet", json!({})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 0})).unwrap(); + let (src, dest) = { + let wb = &s.doc().unwrap().wb; + (wb.sheet(0).unwrap().name.clone(), wb.sheet(1).unwrap().name.clone()) + }; + plant(&mut s, 0, "A1", "a = xl(\"B5\")\nb = xl('A2')"); + plant(&mut s, 0, "A2", "1"); + s.execute("edit.cut", json!({"range": "A1:A2"})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 1})).unwrap(); + s.execute("edit.paste", json!({"at": "C1"})).unwrap(); + // B5 stays on the source sheet; A2 moved with the cut, so it follows to the destination. + assert_eq!(code(&s, 1, "C1").unwrap(), format!("a = xl(\"{src}!B5\")\nb = xl('{dest}!C2')")); + assert_eq!(code(&s, 0, "A1"), None); + // Within one sheet nothing is qualified. + s.execute("edit.cut", json!({"range": "C1"})).unwrap(); + s.execute("edit.paste", json!({"at": "E1"})).unwrap(); + assert_eq!(code(&s, 1, "E1").unwrap(), format!("a = xl(\"{src}!B5\")\nb = xl('{dest}!C2')")); + // A copy to another sheet still carries the code as it is. + plant(&mut s, 0, "A9", "xl(\"B5\")"); + s.execute("sheet.activate", json!({"sheet": 0})).unwrap(); + s.execute("edit.copy", json!({"range": "A9"})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 1})).unwrap(); + s.execute("edit.paste", json!({"at": "A9"})).unwrap(); + assert_eq!(code(&s, 1, "A9").as_deref(), Some("xl(\"B5\")")); +} + +#[test] +fn goal_seek_and_solver_refuse_to_change_a_python_cell() { + let mut s = s(); + plant(&mut s, 0, "A1", "2"); + s.execute("cell.set", json!({"cell": "B1", "input": "=A1*2"})).unwrap(); + let e = s.execute("data.goalSeek", json!({"set": "B1", "to": 10, "changing": "A1"})).unwrap_err(); + assert!(e.to_string().contains("must contain a value"), "{e}"); + assert_eq!(code(&s, 0, "A1").as_deref(), Some("2"), "the code is kept"); + // The set cell must be a formula: a Python cell isn't one this build can recalculate. + s.execute("cell.set", json!({"cell": "C1", "input": "3"})).unwrap(); + assert!(s.execute("data.goalSeek", json!({"set": "A1", "to": 10, "changing": "C1"})).is_err()); + let e = s.execute("data.solver", json!({"objective": "B1", "goal": "max", "variables": "A1"})).unwrap_err(); + assert!(e.to_string().contains("constants"), "{e}"); + assert_eq!(code(&s, 0, "A1").as_deref(), Some("2")); +} + +#[test] +fn text_to_columns_keeps_a_py_piece_as_text() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "'=PY(\"x\");7"})).unwrap(); + assert_eq!(super::v(&s, "A1"), Value::text("=PY(\"x\");7")); + s.execute("data.textToColumns", json!({"range": "A1", "delimiters": [";"]})).unwrap(); + // The default text qualifier takes the quotes away, as in Excel. + assert_eq!(super::v(&s, "A1"), Value::text("=PY(x)"), "the piece is kept, not cleared"); + assert_eq!(code(&s, 0, "A1"), None, "and it is not a Python cell"); + assert_eq!(super::v(&s, "B1"), Value::Number(7.0)); +} + +#[test] +fn python_cells_survive_native_json() { + let mut s = s(); + plant(&mut s, 0, "B2", "x = 1\nx"); + let wb = s.doc().unwrap().wb.clone(); + let json = serde_json::to_string(&*wb).unwrap(); + let back: gridcraft_model::Workbook = serde_json::from_str(&json).unwrap(); + let cell = back.sheet(0).unwrap().cell(CellRef::parse("B2").unwrap()).unwrap(); + assert_eq!(cell.py.as_ref().unwrap().code, "x = 1\nx"); + // Cells written before Python existed have no `py` field. + let plain: Cell = serde_json::from_value(json!({"value": {"t": "Number", "v": 2.0}, "formula": null, "style": 0})).unwrap(); + assert!(plain.py.is_none()); + assert!(!serde_json::to_string(&plain).unwrap().contains("py"), "no field is written for ordinary cells"); +} + +#[test] +fn new_errors_parse_and_have_error_type_numbers() { + use gridcraft_core::CellError; + for (text, e) in + [("#PYTHON!", CellError::Python), ("#BUSY!", CellError::Busy), ("#TIMEOUT!", CellError::Timeout), ("#BLOCKED!", CellError::Blocked)] + { + assert_eq!(CellError::parse(text), Some(e)); + assert_eq!(e.as_str(), text); + } + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "=ERROR.TYPE(#BLOCKED!)"})).unwrap(); + assert_eq!(super::v(&s, "A1"), Value::Number(11.0)); + s.execute("cell.set", json!({"cell": "A2", "input": "=ISERROR(#PYTHON!)"})).unwrap(); + assert_eq!(super::v(&s, "A2"), Value::Bool(true)); +} + +#[test] +fn go_to_special_counts_a_python_cell_as_a_formula_not_a_constant() { + let mut s = s(); + plant(&mut s, 0, "A1", "1 + 1"); + s.execute("cell.set", json!({"cell": "A2", "input": "=1+1"})).unwrap(); + s.execute("cell.set", json!({"cell": "A3", "input": "7"})).unwrap(); + s.execute("selection.set", json!({"range": "A1"})).unwrap(); + let r = s.execute("edit.goToSpecial", json!({"kind": "formulas"})).unwrap(); + assert_eq!(r["count"], 2, "{r}"); + assert_eq!(r["selection"], "A1:A2", "{r}"); + s.execute("selection.set", json!({"range": "A1"})).unwrap(); + let r = s.execute("edit.goToSpecial", json!({"kind": "constants"})).unwrap(); + assert_eq!(r["count"], 1, "{r}"); + assert_eq!(r["selection"], "A3", "{r}"); +} + +#[test] +fn a_macro_counts_python_code_as_data_and_a_blocked_spill_as_no_write() { + let mut s = s(); + // F2 shows 1, its cached value: a step that enters 1 there still replaces the code. + plant(&mut s, 0, "F2", "1"); + // H2's code shows #BLOCKED!, as any other new code there would. + s.execute("python.set", json!({"cell": "H2", "code": "'a'"})).unwrap(); + // J2's result spills over J2:J3. A step that writes J3 blocks it, which doesn't write J2. + s.execute("python.set", json!({"cell": "J2", "code": "[5, 6]"})).unwrap(); + { + let d = s.doc_mut().unwrap(); + let wb = Arc::make_mut(&mut d.wb); + let cell = wb.sheet_mut(0).unwrap().cells.get_mut(CellRef::parse("J2").unwrap()).unwrap(); + let data = vec![Value::Number(5.0), Value::Number(6.0)]; + let value = Value::Array(Arc::new(gridcraft_core::Array::new(2, 1, data).unwrap())); + Arc::make_mut(cell.py.as_mut().unwrap()).output = + Some(gridcraft_model::PyOutput { value, type_name: "list".into(), preview: String::new(), convertible: true }); + } + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(super::v(&s, "J3"), Value::Number(6.0)); + super::macros::put( + &mut s, + json!({"name": "Py", "steps": [ + {"command": "cell.set", "params": {"cell": "F2", "input": "1"}}, + {"command": "python.set", "params": {"cell": "H2", "code": "'b'"}}, + {"command": "cell.set", "params": {"cell": "J3", "input": "x"}} + ]}), + ); + let c = s.execute("macro.check", json!({"name": "Py"})).unwrap(); + assert_eq!(c["footprint"]["sheets"][0]["cells"], json!(["F2", "H2", "J3"]), "{c}"); + assert_eq!(c["footprint"]["sheets"][0]["conflicts"], json!(["F2", "H2"]), "{c}"); + let e = s.execute("macro.run", json!({"name": "Py", "onConflict": "fail"})).unwrap_err().to_string(); + assert!(e.contains("F2") && e.contains("H2") && !e.contains("J2"), "{e}"); + assert_eq!(code(&s, 0, "F2").as_deref(), Some("1"), "nothing was changed"); +} + +#[test] +fn replace_edits_a_python_cells_code_and_keeps_it_a_python_cell() { + let mut s = s(); + plant(&mut s, 0, "A1", "x = 1\nx"); + { + let d = s.doc_mut().unwrap(); + let mut wb = (*d.wb).clone(); + let cell = wb.sheet_mut(0).unwrap().cells.get_mut(CellRef::parse("A1").unwrap()).unwrap(); + Arc::make_mut(cell.py.as_mut().unwrap()).object = true; + d.wb = Arc::new(wb); + } + let r = s.execute("edit.replace", json!({"what": "x", "with": "y"})).unwrap(); + assert_eq!(r["replaced"], 1); + assert_eq!(code(&s, 0, "A1").as_deref(), Some("y = 1\ny")); + let d = s.doc().unwrap(); + let cell = d.wb.sheet(0).unwrap().cell(CellRef::parse("A1").unwrap()).unwrap(); + assert!(cell.py.as_ref().unwrap().object, "the return type is kept"); + assert_eq!(cell.value, Value::Error(gridcraft_core::CellError::Blocked)); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(code(&s, 0, "A1").as_deref(), Some("x = 1\nx")); +} + +#[test] +fn replace_leaves_python_code_that_would_no_longer_fit() { + let mut s = s(); + let long = "a".repeat(gridcraft_model::python::MAX_FORMULA_CHARS / 2); + plant(&mut s, 0, "A1", &long); + let r = s.execute("edit.replace", json!({"what": "a", "with": "bb"})).unwrap(); + assert_eq!(r["replaced"], 0); + assert_eq!(r["skipped"], 1); + assert_eq!(code(&s, 0, "A1").as_deref(), Some(long.as_str())); +} + +#[test] +fn a_hidden_python_cell_on_a_protected_sheet_hides_its_code() { + let mut s = s(); + s.execute("python.set", json!({"cell": "A1", "code": "secret = 1\nsecret"})).unwrap(); + s.execute("python.set", json!({"cell": "B1", "code": "'shown'"})).unwrap(); + s.execute("home.formatCells", json!({"range": "A1", "style": {"protection": {"locked": true, "hidden": true}}})).unwrap(); + let hidden = |s: &Session, a: &str| { + let d = s.doc().unwrap(); + let sh = d.wb.active().unwrap(); + crate::display::formula_hidden(&d.wb, sh, CellRef::parse(a).unwrap()) + }; + assert!(!hidden(&s, "A1"), "an unprotected sheet hides nothing"); + s.execute("review.protectSheet", json!({})).unwrap(); + assert!(hidden(&s, "A1"), "Hidden on a protected sheet hides the code, as it hides a formula"); + assert!(!hidden(&s, "B1")); + // Editing it is refused where a formula's would be: the cell is locked. + assert!(s.execute("python.set", json!({"cell": "A1", "code": "2"})).is_err()); + assert_eq!(code(&s, 0, "A1").as_deref(), Some("secret = 1\nsecret")); +} + +#[test] +fn every_merge_keeps_only_the_upper_left_cells_python_code() { + let mut s = s(); + for (cmd, row) in [("home.mergeAcross", 1), ("home.mergeCells", 2), ("home.mergeCenter", 3)] { + plant(&mut s, 0, &format!("A{row}"), "'kept'"); + plant(&mut s, 0, &format!("B{row}"), "'merged away'"); + s.execute(cmd, json!({"range": format!("A{row}:B{row}")})).unwrap(); + assert_eq!(code(&s, 0, &format!("A{row}")).as_deref(), Some("'kept'"), "{cmd}"); + assert_eq!(code(&s, 0, &format!("B{row}")), None, "{cmd}: the hidden cell is no Python cell"); + assert_eq!(super::v(&s, &format!("B{row}")), Value::Empty, "{cmd}"); + } +} + +#[test] +fn unmerging_lets_a_blocked_python_result_spill_again() { + let mut s = s(); + s.execute("python.set", json!({"cell": "A1", "code": "[5, 6]"})).unwrap(); + { + let d = s.doc_mut().unwrap(); + let wb = Arc::make_mut(&mut d.wb); + let cell = wb.sheet_mut(0).unwrap().cells.get_mut(CellRef::parse("A1").unwrap()).unwrap(); + let value = Value::Array(Arc::new(gridcraft_core::Array::new(2, 1, vec![Value::Number(5.0), Value::Number(6.0)]).unwrap())); + Arc::make_mut(cell.py.as_mut().unwrap()).output = + Some(gridcraft_model::PyOutput { value, type_name: "list".into(), preview: String::new(), convertible: true }); + } + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(super::v(&s, "A2"), Value::Number(6.0)); + s.execute("home.mergeCells", json!({"range": "A2:B2"})).unwrap(); + assert_eq!(super::v(&s, "A1"), Value::Error(gridcraft_core::CellError::Spill), "a merged cell blocks the spill"); + s.execute("home.unmergeCells", json!({"range": "A2:B2"})).unwrap(); + assert_eq!((super::v(&s, "A1"), super::v(&s, "A2")), (Value::Number(5.0), Value::Number(6.0)), "it spills again"); +} + +#[test] +fn save_model_asks_before_it_writes_over_a_python_cell_that_shows_nothing() { + let mut s = s(); + for (cell, input) in [("B1", "=5*A1+4*A2"), ("B2", "=6*A1+4*A2")] { + s.execute("cell.set", json!({"cell": cell, "input": input})).unwrap(); + } + // H2 is a Python cell whose cached value is empty: its code is still content. + { + let d = s.doc_mut().unwrap(); + let wb = Arc::make_mut(&mut d.wb); + let sh = wb.sheet_mut(0).unwrap(); + sh.set_cell( + CellRef::parse("H2").unwrap(), + Cell { value: Value::Empty, formula: None, style: Default::default(), py: Some(Arc::new(PyCode::new("None"))) }, + ); + } + let model = json!({"objective": "B1", "variables": "A1:A2", "constraints": ["B2 <= 24"]}); + let e = s.execute("data.solverSaveModel", json!({"to": "H1", "model": model})).unwrap_err().to_string(); + assert!(e.contains(crate::cmd::analysis::OVERWRITE_ERROR) && e.contains("H2"), "{e}"); + assert_eq!(code(&s, 0, "H2").as_deref(), Some("None"), "nothing was written"); +} + +/// A1 is a Python cell whose kept result `[5, 6]` spills over A1:A2, and B1 reads A2. +pub(super) fn spill_with_a_reader() -> crate::Session { + let mut s = s(); + s.execute("python.set", json!({"cell": "A1", "code": "[5, 6]"})).unwrap(); + { + let d = s.doc_mut().unwrap(); + let wb = Arc::make_mut(&mut d.wb); + let cell = wb.sheet_mut(0).unwrap().cells.get_mut(CellRef::parse("A1").unwrap()).unwrap(); + let value = Value::Array(Arc::new(gridcraft_core::Array::new(2, 1, vec![Value::Number(5.0), Value::Number(6.0)]).unwrap())); + Arc::make_mut(cell.py.as_mut().unwrap()).output = + Some(gridcraft_model::PyOutput { value, type_name: "list".into(), preview: String::new(), convertible: true }); + } + s.execute("formulas.calculateNow", json!({})).unwrap(); + s.execute("cell.set", json!({"cell": "B1", "input": "=SUM(A2)"})).unwrap(); + assert_eq!(super::v(&s, "B1"), Value::Number(6.0)); + s +} + +#[test] +fn new_python_code_recalculates_the_readers_of_its_old_spill() { + let mut s = spill_with_a_reader(); + s.execute("python.set", json!({"cell": "A1", "code": "[7, 8]"})).unwrap(); + assert_eq!(super::v(&s, "A2"), Value::Empty, "new code has no result yet"); + assert_eq!(super::v(&s, "B1"), Value::Number(0.0), "B1 read the old spill"); + // The same code again keeps its result, and its spill. + let mut s = spill_with_a_reader(); + s.execute("python.set", json!({"cell": "A1", "code": "[5, 6]"})).unwrap(); + assert_eq!((super::v(&s, "A2"), super::v(&s, "B1")), (Value::Number(6.0), Value::Number(6.0))); +} + +#[test] +fn replace_in_python_code_recalculates_the_readers_of_its_old_spill() { + let mut s = spill_with_a_reader(); + let r = s.execute("edit.replace", json!({"what": "[5,", "with": "[7,"})).unwrap(); + assert_eq!(r["replaced"], 1); + assert_eq!(super::v(&s, "A2"), Value::Empty); + assert_eq!(super::v(&s, "B1"), Value::Number(0.0), "B1 read the old spill"); +} + +#[test] +fn clearing_pasting_over_or_blocking_a_python_spill_recalculates_its_readers() { + let mut s = spill_with_a_reader(); + s.execute("edit.clearContents", json!({"range": "A1"})).unwrap(); + assert_eq!(super::v(&s, "B1"), Value::Number(0.0), "clear"); + let mut s = spill_with_a_reader(); + s.execute("cell.set", json!({"cell": "D1", "input": "9"})).unwrap(); + s.execute("edit.copy", json!({"range": "D1"})).unwrap(); + s.execute("edit.paste", json!({"at": "A1"})).unwrap(); + assert_eq!(super::v(&s, "B1"), Value::Number(0.0), "paste"); + let mut s = spill_with_a_reader(); + s.execute("home.mergeCells", json!({"range": "A2:A3"})).unwrap(); + assert_eq!(super::v(&s, "A1"), Value::Error(gridcraft_core::CellError::Spill)); + assert_eq!(super::v(&s, "B1"), Value::Number(0.0), "merge"); + s.execute("edit.undo", json!({})).unwrap(); + assert_eq!(super::v(&s, "B1"), Value::Number(6.0), "undo"); + s.execute("edit.redo", json!({})).unwrap(); + assert_eq!(super::v(&s, "B1"), Value::Number(0.0), "redo"); +} + +/// `prefix` and a comment padded to the longest code that still fits in one cell. +fn at_the_limit(prefix: &str) -> String { + let with = |n: usize| format!("{prefix}\n#{}", "x".repeat(n)); + let (mut lo, mut hi) = (0, gridcraft_model::python::MAX_FORMULA_CHARS); + while lo < hi { + let mid = (lo + hi).div_ceil(2); + if PyCode::new(&with(mid)).fits() { lo = mid } else { hi = mid - 1 } + } + with(lo) +} + +#[test] +fn an_edit_that_would_make_python_code_too_long_is_refused() { + let mut s = s(); + s.execute("home.insertSheet", json!({})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 0})).unwrap(); + let other = s.doc().unwrap().wb.sheet(1).unwrap().name.clone(); + let refused = |e: crate::EngineError, cell: &str| { + let e = e.to_string(); + assert!(e.contains(&format!("!{cell} longer than Excel's 8192-character")) && e.contains("nothing was changed"), "{e}"); + }; + // A sheet rename that lengthens a reference. + let a1 = at_the_limit(&format!("a = xl(\"{other}!B2\")")); + plant(&mut s, 0, "A1", &a1); + refused(s.execute("sheet.rename", json!({"sheet": 1, "name": "A much longer sheet name"})).unwrap_err(), "A1"); + assert_eq!(code(&s, 0, "A1").unwrap(), a1); + assert_eq!(s.doc().unwrap().wb.sheet(1).unwrap().name, other); + // One of the same length is made. + let same = "Z".repeat(other.chars().count()); + s.execute("sheet.rename", json!({"sheet": 1, "name": same})).unwrap(); + assert_eq!(code(&s, 0, "A1").unwrap(), a1.replace(&other, &same)); + // Code short of the limit still follows each edit. + plant(&mut s, 0, "A7", "d = xl(\"Z1\")"); + s.execute("selection.set", json!({"range": "B10"})).unwrap(); + s.execute("home.insertColumns", json!({"cols": "Y:Y"})).unwrap(); + assert_eq!(code(&s, 0, "A7").unwrap(), "d = xl(\"AA1\")"); + // An inserted column that turns Z1 into AA1. + let a3 = at_the_limit("b = xl(\"Z1\")"); + plant(&mut s, 0, "A3", &a3); + s.execute("cell.set", json!({"cell": "D1", "input": "kept"})).unwrap(); + s.execute("selection.set", json!({"range": "C1"})).unwrap(); + refused(s.execute("home.insertColumns", json!({"cols": "C:C"})).unwrap_err(), "A3"); + assert_eq!(code(&s, 0, "A3").unwrap(), a3); + assert_eq!(s.doc().unwrap().wb.sheet(0).unwrap().value(CellRef::parse("D1").unwrap()), Value::text("kept")); + // A cut to another sheet, which qualifies B9 with the source sheet. + let a5 = at_the_limit("c = xl(\"B9\")"); + plant(&mut s, 0, "A5", &a5); + s.execute("edit.cut", json!({"range": "A5"})).unwrap(); + s.execute("sheet.activate", json!({"sheet": 1})).unwrap(); + refused(s.execute("edit.paste", json!({"at": "C1"})).unwrap_err(), "C1"); + assert_eq!(code(&s, 0, "A5").unwrap(), a5); + assert_eq!(code(&s, 1, "C1"), None); +} + +/// Makes `cell` a Python cell whose cached value is empty, as a file's `PY` formula with no cached +/// value reads. +fn plant_empty(s: &mut Session, cell: &str) { + plant(s, 0, cell, "x = 1"); + let d = s.doc_mut().unwrap(); + let mut wb = (*d.wb).clone(); + if let Some(c) = wb.sheet_mut(0).unwrap().cells.get_mut(CellRef::parse(cell).unwrap()) { + c.value = Value::Empty; + } + d.wb = Arc::new(wb); +} + +#[test] +fn python_code_with_an_empty_value_counts_as_content() { + // A report or a PivotTable asks before replacing it. + let mut s = s(); + s.execute("range.setValues", json!({"range": "A1", "values": [["Region", "Sales"], ["East", 1], ["West", 2], ["East", 3]]})).unwrap(); + plant_empty(&mut s, "D2"); + let report = json!({"tool": "Descriptive Statistics", "range": "B1:B4", "labels": true, "output": "D1"}); + assert!(s.execute("data.analysisToolPak", report).is_err(), "the report asks for overwrite"); + plant_empty(&mut s, "H4"); + let pivot = s.execute("insert.pivotTable", json!({"destination": "Sheet1!H3", "rows": ["Region"], "values": ["Sales"]})); + assert!(pivot.is_err_and(|e| e.to_string().contains("already data"))); + assert_eq!((code(&s, 0, "D2"), code(&s, 0, "H4")), (Some("x = 1".into()), Some("x = 1".into())), "the code is kept"); + // Cells aren't pushed off the sheet over it. + plant_empty(&mut s, "K1048576"); + assert!(s.execute("home.insertCells", json!({"range": "K1", "shift": "down"})).is_err()); + assert_eq!(code(&s, 0, "K1048576"), Some("x = 1".into())); + // A table without a header row doesn't push it off the sheet either. + s.execute("range.setValues", json!({"range": "X1", "values": [[1], [2]]})).unwrap(); + plant_empty(&mut s, "X1048576"); + assert!(s.execute("insert.table", json!({"range": "X1:X2", "header": false})).is_err()); + assert_eq!(code(&s, 0, "X1048576"), Some("x = 1".into())); + // It is part of the data region. + s.execute("range.setValues", json!({"range": "P1", "values": [[1], [""], [3]]})).unwrap(); + plant_empty(&mut s, "P2"); + let sh = s.doc().unwrap().wb.sheet(0).unwrap().clone(); + assert_eq!(crate::selection::current_region(&sh, CellRef::parse("P1").unwrap()).a1(), "P1:P3"); + // Pasted into part of a legacy array, it is refused like any content. + s.execute("selection.set", json!({"range": "R1:R3"})).unwrap(); + s.execute("cell.set", json!({"input": "=5", "array": true})).unwrap(); + plant_empty(&mut s, "T1"); + s.execute("edit.copy", json!({"range": "T1"})).unwrap(); + assert!(s.execute("edit.paste", json!({"at": "R2"})).is_err_and(|e| e.to_string().contains("part of an array"))); +} diff --git a/crates/engine/src/tests/python_commands.rs b/crates/engine/src/tests/python_commands.rs new file mode 100644 index 000000000..6d0bfac42 --- /dev/null +++ b/crates/engine/src/tests/python_commands.rs @@ -0,0 +1,91 @@ +//! The python.* commands. + +use gridcraft_core::{CellError, Value}; +use serde_json::json; + +use super::{s, v}; + +fn set(s: &mut crate::Session, cell: &str, code: &str) -> serde_json::Value { + s.execute("python.set", json!({"cell": cell, "code": code})).unwrap() +} + +#[test] +fn set_stores_the_code_and_blocks_the_cell_until_it_can_run() { + let mut s = s(); + let r = set(&mut s, "B1", "xl('A1') + 1"); + assert_eq!(r["cell"], "B1"); + assert_eq!(v(&s, "B1"), Value::Error(CellError::Blocked)); + let got = s.execute("cell.get", json!({"cell": "B1"})).unwrap(); + assert_eq!(got["python"]["code"], "xl('A1') + 1"); + assert_eq!(got["python"]["output"], "value"); + assert_eq!(got["input"], "xl('A1') + 1"); + // The active cell by default. + s.execute("selection.set", json!({"cell": "D4"})).unwrap(); + s.execute("python.set", json!({"code": "2"})).unwrap(); + assert_eq!(s.execute("cell.get", json!({"cell": "D4"})).unwrap()["python"]["code"], "2"); + // One undo step takes the cell back. + s.execute("edit.undo", json!({})).unwrap(); + assert!(s.execute("cell.get", json!({"cell": "D4"})).unwrap()["python"].is_null()); +} + +#[test] +fn unchanged_code_keeps_its_cached_result() { + let mut s = s(); + set(&mut s, "A1", "21 * 2"); + // As a file would have cached it. + { + let d = s.doc_mut().unwrap(); + let wb = std::sync::Arc::make_mut(&mut d.wb); + let sh = wb.sheet_mut(0).unwrap(); + let cell = sh.cells.get_mut(gridcraft_core::CellRef::new(0, 0)).unwrap(); + let py = std::sync::Arc::make_mut(cell.py.as_mut().unwrap()); + py.output = Some(gridcraft_model::PyOutput { value: Value::Number(42.0), type_name: "int".into(), preview: "42".into(), convertible: true }); + cell.value = Value::Number(42.0); + } + set(&mut s, "A1", "21 * 2"); + assert_eq!(v(&s, "A1"), Value::Number(42.0), "same code, same result"); + s.execute("python.set", json!({"cell": "A1", "code": "21 * 2", "output": "object"})).unwrap(); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["python"]["output"], "object"); + set(&mut s, "A1", "21 * 3"); + assert_eq!(v(&s, "A1"), Value::Error(CellError::Blocked), "new code has no result yet"); + assert_eq!(s.execute("cell.get", json!({"cell": "A1"})).unwrap()["python"]["output"], "object", "the return type stays"); +} + +#[test] +fn set_refuses_bad_params_long_code_headers_and_protected_cells() { + let mut s = s(); + for p in [ + json!({"cell": "A1"}), + json!({"cell": "A1", "code": 5}), + json!({"cell": "not a cell", "code": "1"}), + json!({"cell": "A1", "code": "1", "output": "sideways"}), + ] { + assert!(s.execute("python.set", p.clone()).is_err(), "{p}"); + } + // The limit is the stored PY(…) formula's 8,192 characters. + assert!(s.execute("python.set", json!({"cell": "A1", "code": "y".repeat(8_080)})).is_err()); + s.execute("python.set", json!({"cell": "A1", "code": "y".repeat(8_079)})).unwrap(); + let quoted = format!("{}{}", "\"".repeat(1_200), "y".repeat(5_800)); + let e = s.execute("python.set", json!({"cell": "A2", "code": quoted})).unwrap_err(); + assert!(e.to_string().contains("8192") || e.to_string().contains("8,192"), "{e}"); + // A table header is the column's name. + s.execute("range.setValues", json!({"range": "C1", "values": [["Name", "Score"], ["a", 1]]})).unwrap(); + s.execute("insert.table", json!({"range": "C1:D2", "headers": true})).unwrap(); + assert!(s.execute("python.set", json!({"cell": "D1", "code": "1"})).is_err()); + assert_eq!(v(&s, "D1"), Value::text("Score")); + set(&mut s, "D2", "1"); + // A locked cell on a protected sheet refuses it. + s.execute("review.protectSheet", json!({})).unwrap(); + let e = s.execute("python.set", json!({"cell": "F1", "code": "1"})).unwrap_err(); + assert!(e.to_string().contains("protected"), "{e}"); +} + +#[test] +fn set_over_a_spilling_formula_takes_the_spill_away() { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "=SEQUENCE(3)"})).unwrap(); + assert_eq!(v(&s, "A3"), Value::Number(3.0)); + set(&mut s, "A1", "[1, 2, 3]"); + assert_eq!(v(&s, "A3"), Value::Empty); + assert!(s.doc().unwrap().wb.sheet(0).unwrap().spill_ranges.is_empty()); +} diff --git a/crates/engine/src/tests/python_files.rs b/crates/engine/src/tests/python_files.rs new file mode 100644 index 000000000..cad3f8745 --- /dev/null +++ b/crates/engine/src/tests/python_files.rs @@ -0,0 +1,744 @@ +//! Python cells in files: native JSON, and xlsx as `_xlfn._xlws.PY(index,0|1)` with the code in +//! the Python part ([MS-XLSX] 2.1.23, 2.3.10) and cached values. Nothing here runs Python: results +//! are cached ones, as a file would hold them. + +use std::io::{Read, Write}; +use std::sync::Arc; + +use gridcraft_core::{Array, CellError, CellRef, RangeRef, Value}; +use gridcraft_model::{Cell, PyCode, PyOutput, Workbook}; +use serde_json::json; + +use super::{s, v}; +use crate::{Session, Trust}; + +fn py(s: &mut Session, cell: &str, code: &str) { + s.execute("python.set", json!({"cell": cell, "code": code})).unwrap(); +} + +/// Gives a Python cell a result, as running it (or a file's cache) would. +fn cache(s: &mut Session, cell: &str, value: Value, type_name: &str) { + let d = s.doc_mut().unwrap(); + let wb = Arc::make_mut(&mut d.wb); + let sh = wb.sheet_mut(0).unwrap(); + let c = sh.cells.get_mut(CellRef::parse(cell).unwrap()).unwrap(); + let p = Arc::make_mut(c.py.as_mut().unwrap()); + p.output = Some(PyOutput { value, type_name: type_name.into(), preview: String::new(), convertible: true }); +} + +fn array(rows: usize, cols: usize, data: Vec) -> Value { + Value::Array(Arc::new(Array::new(rows, cols, data).unwrap())) +} + +fn save(s: &mut Session, format: &str) -> Vec { + let r = s.execute("file.saveBytes", json!({"format": format})).unwrap(); + crate::io::base64_decode(r["base64"].as_str().unwrap()).unwrap() +} + +fn open(s: &mut Session, name: &str, bytes: &[u8]) { + s.execute("file.open", json!({"name": name, "base64": crate::io::base64_encode(bytes)})).unwrap(); +} + +fn part(bytes: &[u8], name: &str) -> String { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut xml = String::new(); + z.by_name(name).unwrap().read_to_string(&mut xml).unwrap(); + xml +} + +fn sheet_xml(bytes: &[u8]) -> String { + part(bytes, "xl/worksheets/sheet1.xml") +} + +/// The package with `edit` applied to each XML part (by name), and `add` added. +fn with_parts(bytes: &[u8], edit: impl Fn(&str, &str) -> String, add: &[(&str, &str)]) -> Vec { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut out = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for i in 0..z.len() { + let mut f = z.by_index(i).unwrap(); + let name = f.name().to_string(); + let mut data = Vec::new(); + f.read_to_end(&mut data).unwrap(); + if let Ok(text) = std::str::from_utf8(&data) { + data = edit(&name, text).into_bytes(); + } + out.start_file(name, zip::write::SimpleFileOptions::default()).unwrap(); + out.write_all(&data).unwrap(); + } + for (name, text) in add { + out.start_file(*name, zip::write::SimpleFileOptions::default()).unwrap(); + out.write_all(text.as_bytes()).unwrap(); + } + out.finish().unwrap().into_inner() +} + +/// The package with `edit` applied to the first sheet's XML. +fn with_sheet_xml(bytes: &[u8], edit: impl Fn(&str) -> String) -> Vec { + with_parts(bytes, |name, text| if name == "xl/worksheets/sheet1.xml" { edit(text) } else { text.to_string() }, &[]) +} + +/// `xml` with its `` replaced by `data`. +fn replace_sheet_data(xml: &str, data: &str) -> String { + match (xml.find(""), xml.find("")) { + (Some(a), Some(b), _) => format!("{}{data}{}", &xml[..a], &xml[b + "".len()..]), + (_, _, Some(a)) => format!("{}{data}{}", &xml[..a], &xml[a + "".len()..]), + _ => panic!("no sheetData in {xml}"), + } +} + +/// An empty workbook's package with `data` as its first sheet's cells and, when `scripts` is +/// given, a Python part holding them (the relationship and the content type included), as +/// [MS-XLSX] 2.1.23 describes it. +fn excel_shaped(data: &str, scripts: Option<&[&str]>) -> Vec { + let mut s = s(); + let plain = save(&mut s, "xlsx"); + let Some(scripts) = scripts else { return with_sheet_xml(&plain, |xml| replace_sheet_data(xml, data)) }; + let codes: String = scripts.iter().map(|c| format!("{c}")).collect(); + let python = format!( + "\n{codes}" + ); + const REL: &str = + ""; + const TYPE: &str = ""; + let edit = |name: &str, text: &str| match name { + "xl/worksheets/sheet1.xml" => replace_sheet_data(text, data), + "xl/_rels/workbook.xml.rels" => text.replace("", &format!("{REL}")), + "[Content_Types].xml" => text.replace("", &format!("{TYPE}")), + _ => text.to_string(), + }; + with_parts(&plain, edit, &[("xl/python/python1.xml", &python)]) +} + +fn code(s: &Session, cell: &str) -> Option { + s.doc().unwrap().wb.active().unwrap().cell(CellRef::parse(cell).unwrap()).and_then(|c| c.py.as_deref()).cloned() +} + +/// A workbook with a value, a formula over a Python cell, a spilling table, an object, long code +/// with quotes and line breaks, and an error. +fn build() -> (Session, String) { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "21"})).unwrap(); + py(&mut s, "B1", "xl('A1') * 2"); + cache(&mut s, "B1", Value::Number(42.0), "int"); + s.execute("cell.set", json!({"cell": "A5", "input": "=B1+1"})).unwrap(); + py(&mut s, "C1", "[[1, 'two'], [3, \"four\"]]"); + cache(&mut s, "C1", array(2, 2, vec![1.0.into(), Value::text("two"), 3.0.into(), Value::text("four")]), "list"); + s.execute("python.set", json!({"cell": "E1", "code": "[1, 2]", "output": "object"})).unwrap(); + cache(&mut s, "E1", array(2, 1, vec![1.0.into(), 2.0.into()]), "list"); + let long = format!("# {}\ntext = \"say \\\"hi\\\"\"\nlen(text)", "x".repeat(7_900)); + assert!(PyCode::new(&long).fits()); + py(&mut s, "F1", &long); + cache(&mut s, "F1", Value::Number(8.0), "int"); + py(&mut s, "G1", "1/0"); + { + let d = s.doc_mut().unwrap(); + let sh = Arc::make_mut(&mut d.wb).sheet_mut(0).unwrap(); + sh.cells.get_mut(CellRef::parse("G1").unwrap()).unwrap().value = Value::Error(CellError::Python); + } + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(v(&s, "B1"), Value::Number(42.0)); + assert_eq!(v(&s, "A5"), Value::Number(43.0)); + assert_eq!(v(&s, "D2"), Value::text("four")); + assert_eq!(v(&s, "E1"), Value::text("list"), "a Python object shows its type"); + assert_eq!(v(&s, "E2"), Value::Empty, "and doesn't spill"); + (s, long) +} + +#[test] +fn xlsx_stores_the_code_in_the_python_part_with_cached_values() { + let (mut s, long) = build(); + let bytes = save(&mut s, "xlsx"); + let xml = sheet_xml(&bytes); + // The formula names the cell's script by index, in the order the cells are written: B1, C1, + // E1, F1, G1. + assert!(xml.contains("_xlfn._xlws.PY(0,0)42"), "{xml}"); + assert!(xml.contains(">_xlfn._xlws.PY(2,1)"), "the Python object cell has return type 1: {xml}"); + assert!(!xml.contains("xl('A1')"), "the code is not in the sheet"); + let python = part(&bytes, "xl/python.xml"); + assert!( + python.contains("xl('A1') * 2"), + "{python}" + ); + assert_eq!(python.matches("").count(), 5); + assert!( + part(&bytes, "[Content_Types].xml").contains("") + ); + assert!( + part(&bytes, "xl/_rels/workbook.xml.rels") + .contains("Type=\"http://schemas.microsoft.com/office/2023/09/relationships/Python\" Target=\"python.xml\"") + ); + assert!(xml.contains("t=\"array\" ref=\"C1:D2\""), "the spill is saved like a dynamic array's: {xml}"); + assert!(xml.contains("#VALUE!") && !xml.contains("#PYTHON!"), "Python errors are cached as #VALUE!"); + + open(&mut s, "book.xlsx", &bytes); + assert_eq!(s.doc().unwrap().trust, Trust::FromFile); + // Cached values show, spills included, and the code is back. + assert_eq!(v(&s, "B1"), Value::Number(42.0)); + assert_eq!(v(&s, "A5"), Value::Number(43.0)); + assert_eq!( + (v(&s, "C1"), v(&s, "D1"), v(&s, "C2"), v(&s, "D2")), + (Value::Number(1.0), Value::text("two"), Value::Number(3.0), Value::text("four")) + ); + assert_eq!(code(&s, "B1").unwrap().code, "xl('A1') * 2"); + assert!(code(&s, "E1").unwrap().object); + assert_eq!(code(&s, "F1").unwrap().code, long, "long code with quotes and line breaks survives"); + assert_eq!(v(&s, "G1"), Value::Error(CellError::Value)); + // A blocker typed into the spill area gives #SPILL!; clearing it spills again, without + // running anything. + s.execute("cell.set", json!({"cell": "D2", "input": "x"})).unwrap(); + assert_eq!(v(&s, "C1"), Value::Error(CellError::Spill)); + s.execute("edit.clearContents", json!({"range": "D2"})).unwrap(); + assert_eq!(v(&s, "D2"), Value::text("four")); + // Saved again, the file holds the same formulas and scripts. + let again = save(&mut s, "xlsx"); + assert!(sheet_xml(&again).contains("_xlfn._xlws.PY(0,0)")); + assert_eq!(part(&again, "xl/python.xml"), python); + // A workbook without Python cells has no Python part. + let mut plain = super::s(); + assert!(!save(&mut plain, "xlsx").windows(10).any(|w| w == b"python.xml")); +} + +#[test] +fn excel_authored_python_cells_are_read_as_python() { + // A workbook as [MS-XLSX] 2.3.10 describes it: `_xlfn._xlws.PY(index,0|1,params…)` naming a + // script of the Python part, whose code names the parameters as `%Px%`, with a spilled result + // cached. + let data = "_xlfn._xlws.PY(0,0)1\ + _xlfn._xlws.PY(1,1,C1:C3)DataFrame\ + 2"; + let bytes = excel_shaped(data, Some(&["[1, 2]", "df = xl(\"%P2%\", headers=True)\ndf.describe()"])); + let mut s = s(); + open(&mut s, "excel.xlsx", &bytes); + let a1 = code(&s, "A1").expect("an Excel Python cell is a Python cell"); + assert_eq!(a1.code, "[1, 2]"); + assert!(!a1.object); + assert_eq!((v(&s, "A1"), v(&s, "A2")), (Value::Number(1.0), Value::Number(2.0)), "its spilled values are kept"); + let cell = s.doc().unwrap().wb.active().unwrap().cell(CellRef::new(0, 0)).unwrap().clone(); + assert!(cell.formula.is_none()); + let b1 = code(&s, "B1").unwrap(); + assert_eq!(b1.code, "df = xl(\"C1:C3\", headers=True)\ndf.describe()", "a parameter is put back as its reference"); + assert!(b1.object); + assert_eq!(v(&s, "B1"), Value::text("DataFrame")); + // Saved by GridCraft, the code goes back into a Python part, the reference as text. + let again = save(&mut s, "xlsx"); + assert!(part(&again, "xl/python.xml").contains("df = xl(\"C1:C3\", headers=True)\ndf.describe()")); + // The string-literal form (other applications, files GridCraft wrote before) still reads. + let literal = "_xlfn._xlws.PY(\"1 + 1\",1)2"; + open(&mut s, "literal.xlsx", &excel_shaped(literal, None)); + assert_eq!(code(&s, "A1").map(|p| (p.code, p.object)), Some(("1 + 1".to_string(), true))); +} + +#[test] +fn a_py_formula_naming_a_missing_script_keeps_only_its_value() { + let data = "_xlfn._xlws.PY(0,0)5_xlfn._xlws.PY(3,0)6"; + for (name, bytes) in [("one.xlsx", excel_shaped(data, Some(&["5"]))), ("none.xlsx", excel_shaped(data, None))] { + let (wb, warnings) = crate::io::open_bytes(name, &bytes).unwrap(); + let sh = wb.sheet(0).unwrap(); + let b1 = sh.cell(CellRef::parse("B1").unwrap()).unwrap(); + assert!(b1.py.is_none() && b1.formula.is_none(), "{name}: no code to keep, and no formula #NAME? would replace"); + assert_eq!(b1.value, Value::Number(6.0), "{name}"); + assert!(warnings.iter().any(|w| w.contains("Python")), "{name}: {warnings:?}"); + } + let (wb, _) = crate::io::open_bytes("one.xlsx", &excel_shaped(data, Some(&["5"]))).unwrap(); + assert_eq!(wb.sheet(0).unwrap().cell(CellRef::parse("A1").unwrap()).unwrap().py.as_ref().map(|p| p.code.as_str()), Some("5")); +} + +/// `bytes` with the part `name` stored uncompressed and one byte of its data changed, so the +/// archive is intact but reading the part fails its checksum. +fn with_damaged_part(bytes: &[u8], name: &str, marker: &str) -> Vec { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut out = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for i in 0..z.len() { + let mut f = z.by_index(i).unwrap(); + let entry = f.name().to_string(); + let mut data = Vec::new(); + f.read_to_end(&mut data).unwrap(); + let options = zip::write::SimpleFileOptions::default(); + let options = if entry == name { options.compression_method(zip::CompressionMethod::Stored) } else { options }; + out.start_file(entry, options).unwrap(); + out.write_all(&data).unwrap(); + } + let mut zipped = out.finish().unwrap().into_inner(); + let at = zipped.windows(marker.len()).position(|w| w == marker.as_bytes()).expect("the marker is stored as it is"); + zipped[at + 1] ^= 0x20; + zipped +} + +#[test] +fn a_damaged_python_part_is_a_warning_and_the_workbook_opens_with_its_values() { + let data = "_xlfn._xlws.PY(0,0)59"; + let good = excel_shaped(data, Some(&["5"])); + // The part's data fails its checksum (a damaged archive entry). + let damaged = with_damaged_part(&good, "xl/python/python1.xml", ""); + // The part is not well-formed XML, before its first script. + let broken = |at: &'static str, with: &'static str| { + with_parts(&good, move |name, text| if name == "xl/python/python1.xml" { text.replace(at, with) } else { text.to_string() }, &[]) + }; + let malformed = broken("", "