diff --git a/crates/calc/src/recalc.rs b/crates/calc/src/recalc.rs index 4cbbcc8b4..e1b1e89cf 100644 --- a/crates/calc/src/recalc.rs +++ b/crates/calc/src/recalc.rs @@ -436,6 +436,10 @@ struct PassHost<'a> { stack_top: usize, /// Whether an evaluation ran past the stack budget since the innermost formula started. out_of_stack: bool, + /// The kept results of the Python cells in the pass: a Python cell isn't evaluated, its + /// result is laid out (an array spills from the cell, as a formula's does). Each one is in + /// `pending` until it is laid out, so a formula that reads it first gets its new value. + python: HashMap, } impl<'a> PassHost<'a> { @@ -468,8 +472,17 @@ impl<'a> PassHost<'a> { taint: false, stack_top: 0, out_of_stack: false, + python: HashMap::default(), } } + + /// Adds the dirty Python cells of the pass and their kept results: like the formulas, they + /// are pending until laid out, so every read path calculates them before reading them. + fn with_python(mut self, python: HashMap) -> Self { + self.pending.extend(python.keys().copied()); + self.python = python; + self + } } /// Whether an array result at `k` can't spill: it is in a table, or its area runs off the sheet, @@ -483,7 +496,10 @@ fn spill_blocked(host: &PassHost<'_>, (sheet, anchor): Key, a: &Array) -> bool { let own = sh.spill_ranges.get(&anchor); let spilled_over = |c: CellRef| { !own.is_some_and(|o| o.contains(c)) - && sh.spill_ranges.iter().any(|(o, r)| *o != anchor && r.contains(c) && !host.exprs.contains_key(&(sheet, *o))) + && sh.spill_ranges.iter().any(|(o, r)| { + // A formula or Python cell recalculated in this pass lays its array out again. + *o != anchor && r.contains(c) && !host.exprs.contains_key(&(sheet, *o)) && !host.python.contains_key(&(sheet, *o)) + }) }; end.row >= gridcraft_core::MAX_ROWS || end.col >= gridcraft_core::MAX_COLS @@ -560,6 +576,9 @@ impl PassHost<'_> { } return v; } + if let Some(result) = self.python.get(&k).cloned() { + return self.python_result(k, result); + } let Some(expr) = self.exprs.get(&k) else { return self.wb.sheet(k.0).map(|s| s.value(k.1)).unwrap_or_default(); }; @@ -677,6 +696,36 @@ impl PassHost<'_> { v } + /// A Python cell's kept result as its value: an array spills from the cell unless it is + /// blocked, malformed or larger than [`gridcraft_model::python::MAX_RESULT_CELLS`] (only a + /// crafted file holds one), which shows `#SPILL!`. + fn python_result(&mut self, k: Key, result: Value) -> Value { + self.pending.remove(&k); + let v = match result { + Value::Array(a) => { + let sane = a.rows >= 1 + && a.cols >= 1 + && a.rows.checked_mul(a.cols).is_some_and(|n| n == a.data.len() && n <= gridcraft_model::python::MAX_RESULT_CELLS); + if sane { + self.spills.insert(k, a.clone()); + a.data.first().cloned().unwrap_or_default() + } else { + Value::Error(CellError::Spill) + } + } + v => v, + }; + let v = match self.spills.get(&k) { + Some(a) if spill_blocked(self, k, a) => { + self.spills.remove(&k); + Value::Error(CellError::Spill) + } + _ => v, + }; + self.results.insert(k, v.clone()); + v + } + /// A formula's previous value for iterative calculation: from the last iteration, else the /// value it shows. Blank, and the `#CIRC!` of a loop calculated with iteration off, are 0. fn previous(&self, k: Key) -> Value { @@ -817,12 +866,12 @@ impl Host for PassHost<'_> { let (h, w) = (range.height() as usize, range.width() as usize); let mut out = vec![Value::Empty; h * w]; let Some(sh) = self.wb.sheet(sheet) else { return out }; - // Sparse walk: only stored cells; formula cells that are (or were) dirty go through - // `compute`, constants are copied straight from the store. + // Sparse walk: only stored cells; formula and Python cells that are (or were) dirty go + // through `compute`, constants are copied straight from the store. let mut formulas: Vec<(usize, CellRef)> = Vec::new(); for (c, cell) in sh.cells.iter_range(range) { let i = (c.row - range.start.row) as usize * w + (c.col - range.start.col) as usize; - if cell.formula.is_some() { + if cell.formula.is_some() || cell.py.is_some() { formulas.push((i, c)); } else if let Some(slot) = out.get_mut(i) { *slot = cell.value.clone(); @@ -888,11 +937,11 @@ impl Host for PassHost<'_> { fn visit_range(&mut self, sheet: usize, range: RangeRef, f: &mut dyn FnMut(&Value) -> bool) -> bool { let wb = self.wb; let Some(sh) = wb.sheet(sheet) else { return false }; - // Formula cells first, in row-major order like `range_values`: reading one calculates it - // and the pass remembers the value, so the walk below gets the same value again without - // this method holding any. + // Formula and Python cells first, in row-major order like `range_values`: reading one + // calculates it (a Python cell lays out its result) and the pass remembers the value, so + // the walk below gets the same value again without this method holding any. for (c, cell) in sh.cells.iter_range(range) { - if cell.formula.is_some() { + if cell.formula.is_some() || cell.py.is_some() { let _ = self.cell_value(sheet, c); } } @@ -902,7 +951,7 @@ impl Host for PassHost<'_> { return false; } for (c, cell) in sh.cells.iter_range(range) { - let go = if cell.formula.is_some() { f(&self.cell_value(sheet, c)) } else { f(&cell.value) }; + let go = if cell.formula.is_some() || cell.py.is_some() { f(&self.cell_value(sheet, c)) } else { f(&cell.value) }; if !go { break; } @@ -953,6 +1002,10 @@ impl Calc { if cell.value == Value::Error(CellError::Circ) { circ.push((si, c)); } + } else if cell.py.is_some() { + // A Python cell lays out its kept result, so it is calculated like a formula + // that reads nothing: its spill is blocked and unblocked like a formula's. + self.graph.insert((si, c), Node { areas: Vec::new(), dynamic: false }); } } } @@ -1074,11 +1127,14 @@ impl Calc { let mut cleared: Vec<(usize, RangeRef)> = Vec::new(); for &k in changed { let formula = wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.formula.clone()); + let python = wb.sheet(k.0).and_then(|s| s.cell(k.1)).is_some_and(|c| c.py.is_some()); match formula.and_then(|f| f.expr()) { Some(e) => { let (areas, dynamic) = precedents(wb, k.0, &e); self.graph.insert(k, Node { areas, dynamic }); } + // See `rebuild`: its spill is laid out again from its result when it is calculated. + None if python => self.graph.insert(k, Node { areas: Vec::new(), dynamic: false }), None => { self.graph.remove(k); // A formula replaced by a constant (or cleared) takes its spilled values with it. @@ -1206,11 +1262,15 @@ impl Calc { exprs.insert(*k, e); } } + let python: HashMap = dirty + .iter() + .filter_map(|k| wb.sheet(k.0).and_then(|s| s.cell(k.1)).and_then(|c| c.py.as_ref()).and_then(|p| p.result_value()).map(|v| (*k, v))) + .collect(); let now = self.now(); let mut rng = self.rng; let tp = prof_now(); let (results, spills, cycles) = { - let mut host = PassHost::new(wb, &exprs, &mut rng, now); + let mut host = PassHost::new(wb, &exprs, &mut rng, now).with_python(python); host.iterate = wb.calc.iterative; for k in &dirty { host.calculate(*k); diff --git a/crates/calc/src/tests.rs b/crates/calc/src/tests.rs index 5b9e90085..1f68a58fc 100644 --- a/crates/calc/src/tests.rs +++ b/crates/calc/src/tests.rs @@ -2274,3 +2274,63 @@ fn loan_schedule_residue_keeps_excels_sign() { } assert_eq!(t.get("P7"), Value::Text("".into())); } + +/// Makes `at` a Python cell showing `cached` (its value before this calculation), with `result` +/// as its kept result, and returns the cell's key for `Calc::cells_changed`. +fn plant_python(t: &mut T, at: &str, cached: Value, result: Value) -> (usize, CellRef) { + use gridcraft_model::python::{PyCode, PyOutput}; + let output = PyOutput { value: result, type_name: "list".into(), preview: String::new(), convertible: true }; + let py = PyCode { output: Some(output), ..PyCode::new("[5, 6]") }; + let cell = Cell { value: cached, formula: None, style: gridcraft_model::StyleId::default(), py: Some(std::sync::Arc::new(py)) }; + t.wb.sheet_mut(0).unwrap().cells.set(c(at), cell); + (0, c(at)) +} + +fn column(values: &[f64]) -> Value { + let data = values.iter().map(|v| Value::Number(*v)).collect(); + Value::Array(std::sync::Arc::new(gridcraft_core::Array::new(values.len(), 1, data).unwrap())) +} + +#[test] +fn formulas_read_a_dirty_python_cell_after_it_is_laid_out() { + // B1's result can't spill (B2 holds a value), so it is #SPILL!. A1, A2 and A3 come before + // it in the pass; each must lay it out first rather than read its old value, 1. + let build = |t: &mut T| { + t.set("B2", "10"); + t.set("B3", "100"); + t.set("A1", "=B1"); + t.set("A2", "=MAX(B1:B3)"); + t.set("A3", "=SUM(B1:B3)"); + t.set("A4", "=COUNTA(B1:B3)"); + t.set("A5", "=SUMPRODUCT(--ISERROR(B1:B3))"); + }; + let check = |t: &T, how: &str| { + assert_eq!(t.get("B1"), Value::Error(CellError::Spill), "{how}"); + assert_eq!(t.get("A1"), Value::Error(CellError::Spill), "{how}: a reference"); + assert_eq!(t.get("A2"), Value::Error(CellError::Spill), "{how}: a range read"); + assert_eq!(t.get("A3"), Value::Error(CellError::Spill), "{how}: a range walked cell by cell"); + assert_eq!(t.num("A4"), 3.0, "{how}"); + assert_eq!(t.num("A5"), 1.0, "{how}: the error is seen in the range"); + }; + + // A full recalculation. + let mut t = T::new(); + build(&mut t); + plant_python(&mut t, "B1", Value::Number(1.0), column(&[5.0, 6.0])); + t.calc.recalc_all(&mut t.wb); + check(&t, "full"); + + // An incremental one: B1 becomes a Python cell after its readers. + let mut t = T::new(); + t.set("B1", "1"); + build(&mut t); + assert_eq!((t.num("A1"), t.num("A3")), (1.0, 111.0)); + let k = plant_python(&mut t, "B1", Value::Number(1.0), column(&[5.0, 6.0])); + t.calc.cells_changed(&mut t.wb, &[k]); + check(&t, "incremental"); + + // Unblocked, the readers see the new array. + t.set("B2", ""); + assert_eq!((t.num("B1"), t.num("B2")), (5.0, 6.0)); + assert_eq!((t.num("A1"), t.num("A2"), t.num("A3")), (5.0, 100.0, 111.0)); +} diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs index 4ad9f3d83..a65139dd6 100644 --- a/crates/engine/src/io.rs +++ b/crates/engine/src/io.rs @@ -119,7 +119,139 @@ fn decrypt_message(name: &str, e: gridcraft_xlsx::DecryptError) -> String { } /// Parses bytes that aren't encrypted. +/// Parses bytes that aren't encrypted, and adopts the Python cells they hold (see +/// [`adopt_python`]) within the limits of [`enforce_python_limits`]. fn open_plain(name: &str, bytes: &[u8], csv_as_text: bool) -> Result<(Workbook, Vec)> { + let (mut wb, mut warnings) = read_plain(name, bytes, csv_as_text)?; + warnings.extend(adopt_python(&mut wb)); + warnings.extend(enforce_python_limits(&mut wb)); + Ok((wb, warnings)) +} + +/// The most result cells [`adopt_python`] rebuilds from a file's cached spills, over all its +/// Python cells (each one is also limited to [`gridcraft_model::python::MAX_RESULT_CELLS`]). +pub const MAX_ADOPTED_RESULT_CELLS: u64 = 4_000_000; + +/// Python cells from a file: a cell whose formula is `PY("…",0|1)` (a format that kept it as a +/// formula) becomes a Python cell, and a Python cell whose spilled values the file cached gets +/// them back as its result, so it shows them until it runs again. Results are rebuilt for at most +/// [`MAX_ADOPTED_RESULT_CELLS`] cells in all; the rest keep only their own cached value. Returns +/// the warnings. +pub fn adopt_python(wb: &mut Workbook) -> Vec { + let mut budget = MAX_ADOPTED_RESULT_CELLS; + let mut skipped = 0usize; + for si in 0..wb.sheets.len() { + let Some(sh) = wb.sheets.get(si) else { continue }; + let formulas: Vec<(CellRef, gridcraft_model::PyCode)> = sh + .cells + .iter() + .filter_map(|(c, cell)| cell.formula.as_ref().and_then(|f| gridcraft_model::PyCode::from_formula(&f.text)).map(|p| (c, p))) + .collect(); + let max = gridcraft_model::python::MAX_RESULT_CELLS as u64; + let spills: Vec<(CellRef, RangeRef)> = sh + .spill_ranges + .iter() + .filter(|(a, r)| r.count() <= max && sh.cell(**a).and_then(|c| c.py.as_ref()).is_some_and(|p| p.output.is_none())) + .map(|(a, r)| (*a, *r)) + .collect(); + if formulas.is_empty() && spills.is_empty() { + continue; + } + let Some(sh) = wb.sheet_mut(si) else { continue }; + for (c, py) in formulas { + if let Some(cell) = sh.cells.get_mut(c) { + cell.formula = None; + cell.py = Some(std::sync::Arc::new(py)); + } + } + for (anchor, range) in spills { + let Some(left) = budget.checked_sub(range.count()) else { + skipped += 1; + continue; + }; + budget = left; + let data: Vec = + range.iter().map(|c| if c == anchor { sh.cell(c).map(|x| x.value.clone()).unwrap_or_default() } else { sh.value(c) }).collect(); + let Some(array) = gridcraft_core::Array::new(range.height() as usize, range.width() as usize, data) else { continue }; + if let Some(py) = sh.cells.get_mut(anchor).and_then(|x| x.py.as_mut()) { + std::sync::Arc::make_mut(py).output = + Some(gridcraft_model::PyOutput { value: array.into(), type_name: "list".into(), preview: String::new(), convertible: true }); + } + } + } + if skipped == 0 { + return Vec::new(); + } + vec![format!( + "The file's Python results are larger than {MAX_ADOPTED_RESULT_CELLS} cells in all, so {skipped} Python cell(s) show only their first value until they run again." + )] +} + +/// A file is somebody else's work: a Python cell whose code doesn't fit in Excel's +/// 8,192-character formula keeps only its cached value, and a kept result that is not a +/// well-formed array of at most [`gridcraft_model::python::MAX_RESULT_CELLS`] cells is dropped +/// (the cell keeps its value). Returns the warnings. +pub fn enforce_python_limits(wb: &mut Workbook) -> Vec { + let mut long = Vec::new(); + for si in 0..wb.sheets.len() { + let Some(sh) = wb.sheets.get(si) else { continue }; + let mut over = Vec::new(); + let mut bad_output = Vec::new(); + for (c, cell) in sh.cells.iter() { + let Some(py) = &cell.py else { continue }; + if !py.fits() { + long.push(format!("{}!{}", sh.name, c.a1())); + over.push(c); + } else if py.output.as_ref().is_some_and(|o| !sane_python_output(&o.value)) { + bad_output.push(c); + } + } + if over.is_empty() && bad_output.is_empty() { + continue; + } + let dropped: Vec = over.iter().chain(&bad_output).copied().collect(); + let Some(sh) = wb.sheet_mut(si) else { continue }; + for c in over { + if let Some(cell) = sh.cells.get_mut(c) { + cell.py = None; + } + } + for c in bad_output { + if let Some(py) = sh.cells.get_mut(c).and_then(|x| x.py.as_mut()) { + std::sync::Arc::make_mut(py).output = None; + } + } + // The spill of a dropped result (or of code dropped to its value) goes with it, so the + // sheet and a save no longer show the old spilled values. + for c in dropped { + if let Some(r) = sh.spill_ranges.remove(&c) { + sh.clear_spill(r); + } + } + } + let Some(first) = long.first() else { return vec![] }; + let more = if long.len() > 1 { format!(" (and {} more)", long.len() - 1) } else { String::new() }; + vec![format!( + "A Python cell's code is longer than Excel's {}-character formula limit, so {first}{more} kept only its last calculated value.", + gridcraft_model::python::MAX_FORMULA_CHARS + )] +} + +/// A Python result a file may hold: a scalar, or an array whose size matches its data and has at +/// most [`gridcraft_model::python::MAX_RESULT_CELLS`] cells. +fn sane_python_output(v: &gridcraft_core::Value) -> bool { + match v { + gridcraft_core::Value::Array(a) => { + a.rows >= 1 + && a.cols >= 1 + && a.rows.checked_mul(a.cols).is_some_and(|n| n == a.data.len() && n <= gridcraft_model::python::MAX_RESULT_CELLS) + } + _ => true, + } +} + +/// Parses bytes that aren't encrypted, as they are. +fn read_plain(name: &str, bytes: &[u8], csv_as_text: bool) -> Result<(Workbook, Vec)> { let kind = FileKind::from_path(name); let sniffed = gridcraft_xlsx::sniff(bytes); if sniffed == gridcraft_xlsx::Format::Xls { diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 47eada0b4..c0adf6002 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5095,6 +5095,7 @@ fn a_script_cannot_switch_windows() { mod python_cells; mod python_commands; +mod python_files; mod seams; #[test] diff --git a/crates/engine/src/tests/python_files.rs b/crates/engine/src/tests/python_files.rs new file mode 100644 index 000000000..00b597445 --- /dev/null +++ b/crates/engine/src/tests/python_files.rs @@ -0,0 +1,441 @@ +//! Python cells in files: native JSON, and xlsx as `_xlfn._xlws.PY(index,0|1)` with the code in +//! the Python part ([MS-XLSX] 2.1.23, 2.3.10) and cached values. Nothing here runs Python: results +//! are cached ones, as a file would hold them. + +use std::io::{Read, Write}; +use std::sync::Arc; + +use gridcraft_core::{Array, CellError, CellRef, RangeRef, Value}; +use gridcraft_model::{Cell, PyCode, PyOutput, Workbook}; +use serde_json::json; + +use super::{s, v}; +use crate::{Session, Trust}; + +fn py(s: &mut Session, cell: &str, code: &str) { + s.execute("python.set", json!({"cell": cell, "code": code})).unwrap(); +} + +/// Gives a Python cell a result, as running it (or a file's cache) would. +fn cache(s: &mut Session, cell: &str, value: Value, type_name: &str) { + let d = s.doc_mut().unwrap(); + let wb = Arc::make_mut(&mut d.wb); + let sh = wb.sheet_mut(0).unwrap(); + let c = sh.cells.get_mut(CellRef::parse(cell).unwrap()).unwrap(); + let p = Arc::make_mut(c.py.as_mut().unwrap()); + p.output = Some(PyOutput { value, type_name: type_name.into(), preview: String::new(), convertible: true }); +} + +fn array(rows: usize, cols: usize, data: Vec) -> Value { + Value::Array(Arc::new(Array::new(rows, cols, data).unwrap())) +} + +fn save(s: &mut Session, format: &str) -> Vec { + let r = s.execute("file.saveBytes", json!({"format": format})).unwrap(); + crate::io::base64_decode(r["base64"].as_str().unwrap()).unwrap() +} + +fn open(s: &mut Session, name: &str, bytes: &[u8]) { + s.execute("file.open", json!({"name": name, "base64": crate::io::base64_encode(bytes)})).unwrap(); +} + +fn part(bytes: &[u8], name: &str) -> String { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut xml = String::new(); + z.by_name(name).unwrap().read_to_string(&mut xml).unwrap(); + xml +} + +fn sheet_xml(bytes: &[u8]) -> String { + part(bytes, "xl/worksheets/sheet1.xml") +} + +/// The package with `edit` applied to each XML part (by name), and `add` added. +fn with_parts(bytes: &[u8], edit: impl Fn(&str, &str) -> String, add: &[(&str, &str)]) -> Vec { + let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); + let mut out = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for i in 0..z.len() { + let mut f = z.by_index(i).unwrap(); + let name = f.name().to_string(); + let mut data = Vec::new(); + f.read_to_end(&mut data).unwrap(); + if let Ok(text) = std::str::from_utf8(&data) { + data = edit(&name, text).into_bytes(); + } + out.start_file(name, zip::write::SimpleFileOptions::default()).unwrap(); + out.write_all(&data).unwrap(); + } + for (name, text) in add { + out.start_file(*name, zip::write::SimpleFileOptions::default()).unwrap(); + out.write_all(text.as_bytes()).unwrap(); + } + out.finish().unwrap().into_inner() +} + +/// The package with `edit` applied to the first sheet's XML. +fn with_sheet_xml(bytes: &[u8], edit: impl Fn(&str) -> String) -> Vec { + with_parts(bytes, |name, text| if name == "xl/worksheets/sheet1.xml" { edit(text) } else { text.to_string() }, &[]) +} + +/// `xml` with its `` replaced by `data`. +fn replace_sheet_data(xml: &str, data: &str) -> String { + match (xml.find(""), xml.find("")) { + (Some(a), Some(b), _) => format!("{}{data}{}", &xml[..a], &xml[b + "".len()..]), + (_, _, Some(a)) => format!("{}{data}{}", &xml[..a], &xml[a + "".len()..]), + _ => panic!("no sheetData in {xml}"), + } +} + +/// An empty workbook's package with `data` as its first sheet's cells and, when `scripts` is +/// given, a Python part holding them (the relationship and the content type included), as +/// [MS-XLSX] 2.1.23 describes it. +fn excel_shaped(data: &str, scripts: Option<&[&str]>) -> Vec { + let mut s = s(); + let plain = save(&mut s, "xlsx"); + let Some(scripts) = scripts else { return with_sheet_xml(&plain, |xml| replace_sheet_data(xml, data)) }; + let codes: String = scripts.iter().map(|c| format!("{c}")).collect(); + let python = format!( + "\n{codes}" + ); + const REL: &str = + ""; + const TYPE: &str = ""; + let edit = |name: &str, text: &str| match name { + "xl/worksheets/sheet1.xml" => replace_sheet_data(text, data), + "xl/_rels/workbook.xml.rels" => text.replace("", &format!("{REL}")), + "[Content_Types].xml" => text.replace("", &format!("{TYPE}")), + _ => text.to_string(), + }; + with_parts(&plain, edit, &[("xl/python/python1.xml", &python)]) +} + +fn code(s: &Session, cell: &str) -> Option { + s.doc().unwrap().wb.active().unwrap().cell(CellRef::parse(cell).unwrap()).and_then(|c| c.py.as_deref()).cloned() +} + +/// A workbook with a value, a formula over a Python cell, a spilling table, an object, long code +/// with quotes and line breaks, and an error. +fn build() -> (Session, String) { + let mut s = s(); + s.execute("cell.set", json!({"cell": "A1", "input": "21"})).unwrap(); + py(&mut s, "B1", "xl('A1') * 2"); + cache(&mut s, "B1", Value::Number(42.0), "int"); + s.execute("cell.set", json!({"cell": "A5", "input": "=B1+1"})).unwrap(); + py(&mut s, "C1", "[[1, 'two'], [3, \"four\"]]"); + cache(&mut s, "C1", array(2, 2, vec![1.0.into(), Value::text("two"), 3.0.into(), Value::text("four")]), "list"); + s.execute("python.set", json!({"cell": "E1", "code": "[1, 2]", "output": "object"})).unwrap(); + cache(&mut s, "E1", array(2, 1, vec![1.0.into(), 2.0.into()]), "list"); + let long = format!("# {}\ntext = \"say \\\"hi\\\"\"\nlen(text)", "x".repeat(7_900)); + assert!(PyCode::new(&long).fits()); + py(&mut s, "F1", &long); + cache(&mut s, "F1", Value::Number(8.0), "int"); + py(&mut s, "G1", "1/0"); + { + let d = s.doc_mut().unwrap(); + let sh = Arc::make_mut(&mut d.wb).sheet_mut(0).unwrap(); + sh.cells.get_mut(CellRef::parse("G1").unwrap()).unwrap().value = Value::Error(CellError::Python); + } + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(v(&s, "B1"), Value::Number(42.0)); + assert_eq!(v(&s, "A5"), Value::Number(43.0)); + assert_eq!(v(&s, "D2"), Value::text("four")); + assert_eq!(v(&s, "E1"), Value::text("list"), "a Python object shows its type"); + assert_eq!(v(&s, "E2"), Value::Empty, "and doesn't spill"); + (s, long) +} + +#[test] +fn xlsx_stores_the_code_in_the_python_part_with_cached_values() { + let (mut s, long) = build(); + let bytes = save(&mut s, "xlsx"); + let xml = sheet_xml(&bytes); + // The formula names the cell's script by index, in the order the cells are written: B1, C1, + // E1, F1, G1. + assert!(xml.contains("_xlfn._xlws.PY(0,0)42"), "{xml}"); + assert!(xml.contains(">_xlfn._xlws.PY(2,1)"), "the Python object cell has return type 1: {xml}"); + assert!(!xml.contains("xl('A1')"), "the code is not in the sheet"); + let python = part(&bytes, "xl/python.xml"); + assert!( + python.contains("xl('A1') * 2"), + "{python}" + ); + assert_eq!(python.matches("").count(), 5); + assert!( + part(&bytes, "[Content_Types].xml").contains("") + ); + assert!( + part(&bytes, "xl/_rels/workbook.xml.rels") + .contains("Type=\"http://schemas.microsoft.com/office/2023/09/relationships/Python\" Target=\"python.xml\"") + ); + assert!(xml.contains("t=\"array\" ref=\"C1:D2\""), "the spill is saved like a dynamic array's: {xml}"); + assert!(xml.contains("#VALUE!") && !xml.contains("#PYTHON!"), "Python errors are cached as #VALUE!"); + + open(&mut s, "book.xlsx", &bytes); + assert_eq!(s.doc().unwrap().trust, Trust::FromFile); + // Cached values show, spills included, and the code is back. + assert_eq!(v(&s, "B1"), Value::Number(42.0)); + assert_eq!(v(&s, "A5"), Value::Number(43.0)); + assert_eq!( + (v(&s, "C1"), v(&s, "D1"), v(&s, "C2"), v(&s, "D2")), + (Value::Number(1.0), Value::text("two"), Value::Number(3.0), Value::text("four")) + ); + assert_eq!(code(&s, "B1").unwrap().code, "xl('A1') * 2"); + assert!(code(&s, "E1").unwrap().object); + assert_eq!(code(&s, "F1").unwrap().code, long, "long code with quotes and line breaks survives"); + assert_eq!(v(&s, "G1"), Value::Error(CellError::Value)); + // A blocker typed into the spill area gives #SPILL!; clearing it spills again, without + // running anything. + s.execute("cell.set", json!({"cell": "D2", "input": "x"})).unwrap(); + assert_eq!(v(&s, "C1"), Value::Error(CellError::Spill)); + s.execute("edit.clearContents", json!({"range": "D2"})).unwrap(); + assert_eq!(v(&s, "D2"), Value::text("four")); + // Saved again, the file holds the same formulas and scripts. + let again = save(&mut s, "xlsx"); + assert!(sheet_xml(&again).contains("_xlfn._xlws.PY(0,0)")); + assert_eq!(part(&again, "xl/python.xml"), python); + // A workbook without Python cells has no Python part. + let mut plain = super::s(); + assert!(!save(&mut plain, "xlsx").windows(10).any(|w| w == b"python.xml")); +} + +#[test] +fn excel_authored_python_cells_are_read_as_python() { + // A workbook as [MS-XLSX] 2.3.10 describes it: `_xlfn._xlws.PY(index,0|1,params…)` naming a + // script of the Python part, whose code names the parameters as `%Px%`, with a spilled result + // cached. + let data = "_xlfn._xlws.PY(0,0)1\ + _xlfn._xlws.PY(1,1,C1:C3)DataFrame\ + 2"; + let bytes = excel_shaped(data, Some(&["[1, 2]", "df = xl(\"%P2%\", headers=True)\ndf.describe()"])); + let mut s = s(); + open(&mut s, "excel.xlsx", &bytes); + let a1 = code(&s, "A1").expect("an Excel Python cell is a Python cell"); + assert_eq!(a1.code, "[1, 2]"); + assert!(!a1.object); + assert_eq!((v(&s, "A1"), v(&s, "A2")), (Value::Number(1.0), Value::Number(2.0)), "its spilled values are kept"); + let cell = s.doc().unwrap().wb.active().unwrap().cell(CellRef::new(0, 0)).unwrap().clone(); + assert!(cell.formula.is_none()); + let b1 = code(&s, "B1").unwrap(); + assert_eq!(b1.code, "df = xl(\"C1:C3\", headers=True)\ndf.describe()", "a parameter is put back as its reference"); + assert!(b1.object); + assert_eq!(v(&s, "B1"), Value::text("DataFrame")); + // Saved by GridCraft, the code goes back into a Python part, the reference as text. + let again = save(&mut s, "xlsx"); + assert!(part(&again, "xl/python.xml").contains("df = xl(\"C1:C3\", headers=True)\ndf.describe()")); + // The string-literal form (other applications, files GridCraft wrote before) still reads. + let literal = "_xlfn._xlws.PY(\"1 + 1\",1)2"; + open(&mut s, "literal.xlsx", &excel_shaped(literal, None)); + assert_eq!(code(&s, "A1").map(|p| (p.code, p.object)), Some(("1 + 1".to_string(), true))); +} + +#[test] +fn a_py_formula_naming_a_missing_script_keeps_only_its_value() { + let data = "_xlfn._xlws.PY(0,0)5_xlfn._xlws.PY(3,0)6"; + for (name, bytes) in [("one.xlsx", excel_shaped(data, Some(&["5"]))), ("none.xlsx", excel_shaped(data, None))] { + let (wb, warnings) = crate::io::open_bytes(name, &bytes).unwrap(); + let sh = wb.sheet(0).unwrap(); + let b1 = sh.cell(CellRef::parse("B1").unwrap()).unwrap(); + assert!(b1.py.is_none() && b1.formula.is_none(), "{name}: no code to keep, and no formula #NAME? would replace"); + assert_eq!(b1.value, Value::Number(6.0), "{name}"); + assert!(warnings.iter().any(|w| w.contains("Python")), "{name}: {warnings:?}"); + } + let (wb, _) = crate::io::open_bytes("one.xlsx", &excel_shaped(data, Some(&["5"]))).unwrap(); + assert_eq!(wb.sheet(0).unwrap().cell(CellRef::parse("A1").unwrap()).unwrap().py.as_ref().map(|p| p.code.as_str()), Some("5")); +} + +#[test] +fn many_cells_naming_one_long_script_take_from_one_budget() { + // 20,000 cells name one script of 8,000 characters: 160 MB of code if each took a copy. + let long = "x".repeat(8_000); + let mut data = String::from(""); + for r in 1..=20_000 { + data.push_str(&format!("_xlfn._xlws.PY(0,0)1")); + } + data.push_str(""); + let (wb, warnings) = crate::io::open_bytes("many.xlsx", &excel_shaped(&data, Some(&[&long]))).unwrap(); + let sh = wb.sheet(0).unwrap(); + let with_code = sh.cells.iter().filter(|(_, c)| c.py.is_some()).count(); + assert_eq!(with_code, 64 * 1024 * 1024 / 8_000); + assert!(warnings.iter().any(|w| w.contains("Python")), "{warnings:?}"); + assert_eq!(sh.value(CellRef::parse("A20000").unwrap()), Value::Number(1.0), "the others keep their values"); +} + +#[test] +fn native_json_keeps_python_cells_and_their_results() { + let (mut s, long) = build(); + let bytes = save(&mut s, "json"); + open(&mut s, "book.json", &bytes); + assert_eq!(s.doc().unwrap().trust, Trust::FromFile); + assert_eq!(v(&s, "B1"), Value::Number(42.0)); + assert_eq!(v(&s, "D2"), Value::text("four")); + assert_eq!(v(&s, "G1"), Value::Error(CellError::Python), "the native format keeps the exact error"); + assert_eq!(code(&s, "F1").unwrap().code, long); + let e1 = code(&s, "E1").unwrap(); + assert!(e1.object); + assert_eq!(e1.output.unwrap().type_name, "list"); +} + +#[test] +fn a_spill_range_claiming_the_whole_sheet_is_dropped_on_open() { + let (mut s, _) = build(); + let bytes = save(&mut s, "xlsx"); + let evil = with_sheet_xml(&bytes, |xml| xml.replace("ref=\"C1:D2\"", "ref=\"C1:XFD1048576\"")); + assert_ne!(evil, bytes); + // Walking the claimed range would take hours; the test finishing is the check. + open(&mut s, "evil.xlsx", &evil); + let sh = s.doc().unwrap().wb.active().unwrap(); + assert!(sh.spill_ranges.values().all(|r| r.count() <= gridcraft_model::python::MAX_RESULT_CELLS as u64), "{:?}", sh.spill_ranges); + // Edits afterwards don't walk the claimed range either. + s.execute("cell.set", json!({"cell": "H9", "input": "1"})).unwrap(); + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(v(&s, "C1"), Value::Number(1.0), "the anchor keeps its cached value"); +} + +#[test] +fn rebuilt_results_have_a_budget_across_the_workbook() { + // Five Python cells each claiming a 1,000 x 1,000 spill: more than the budget in all. + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + for i in 0..5u32 { + let anchor = CellRef::new(0, i * 1_000); + sh.set_cell(anchor, Cell { py: Some(Arc::new(PyCode::new("x"))), ..Cell::value(Value::Number(1.0)) }); + sh.spill_ranges.insert(anchor, RangeRef::new(anchor, CellRef::new(999, i * 1_000 + 999))); + } + let warnings = crate::io::adopt_python(&mut wb); + let sh = wb.sheet(0).unwrap(); + let rebuilt = sh.cells.iter().filter(|(_, c)| c.py.as_ref().is_some_and(|p| p.output.is_some())).count(); + assert_eq!(rebuilt as u64, crate::io::MAX_ADOPTED_RESULT_CELLS / 1_000_000); + assert!(warnings.iter().any(|w| w.contains("Python")), "{warnings:?}"); +} + +#[test] +fn hostile_python_content_in_files_is_capped() { + let (mut s, _) = build(); + let bytes = save(&mut s, "json"); + // Make the Python cells hostile: code over the limit, and a result whose shape claims far + // more cells than it holds (the array fields are public, so a crafted file can say anything). + let mut wb: Workbook = serde_json::from_slice(&bytes).unwrap(); + let sh = wb.sheet_mut(0).unwrap(); + let b1 = sh.cells.get_mut(CellRef::parse("B1").unwrap()).unwrap(); + Arc::make_mut(b1.py.as_mut().unwrap()).code = "y".repeat(9_000); + let c1 = sh.cells.get_mut(CellRef::parse("C1").unwrap()).unwrap(); + let out = Arc::make_mut(c1.py.as_mut().unwrap()).output.as_mut().unwrap(); + out.value = Value::Array(Arc::new(Array { rows: 100_000, cols: 100_000, data: vec![Value::Number(1.0)] })); + let bytes = serde_json::to_vec(&wb).unwrap(); + let (wb, warnings) = crate::io::open_bytes("evil.json", &bytes).unwrap(); + let sh = wb.sheet(0).unwrap(); + let b1 = sh.cell(CellRef::parse("B1").unwrap()).unwrap(); + assert!(b1.py.is_none(), "code over the formula limit is dropped"); + assert_eq!(b1.value, Value::Number(42.0), "the cached value stays"); + assert!(warnings.iter().any(|w| w.contains("Python cell")), "{warnings:?}"); + let c1 = sh.cell(CellRef::parse("C1").unwrap()).unwrap(); + assert!(c1.py.as_ref().unwrap().output.is_none(), "a malformed result is dropped"); + open(&mut s, "evil.json", &bytes); + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(v(&s, "C1"), Value::Number(1.0)); + + // PY formulas from other formats become Python cells; non-literal PY calls stay formulas. + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + sh.set_cell(CellRef::new(0, 0), Cell::formula(gridcraft_model::Formula::new("=PY(\"1+\"&\"1\",1)"))); + sh.set_cell(CellRef::new(1, 0), Cell::formula(gridcraft_model::Formula::new("=PY(A1,0)"))); + sh.set_cell(CellRef::new(2, 0), Cell::formula(gridcraft_model::Formula::new("=_xlfn._xlws.PY(\"2\",0)"))); + crate::io::adopt_python(&mut wb); + let sh = wb.sheet(0).unwrap(); + let a1 = sh.cell(CellRef::new(0, 0)).unwrap(); + assert_eq!(a1.py.as_ref().map(|p| (p.code.as_str(), p.object)), Some(("1+1", true))); + assert!(a1.formula.is_none()); + assert!(sh.cell(CellRef::new(1, 0)).unwrap().py.is_none()); + assert_eq!(sh.cell(CellRef::new(2, 0)).unwrap().py.as_ref().map(|p| p.code.as_str()), Some("2")); +} + +#[test] +fn a_malformed_result_never_spills() { + let mut s = s(); + py(&mut s, "A1", "x"); + let bad = Array { rows: 4_000_000_000, cols: 4_000_000_000, data: vec![] }; + cache(&mut s, "A1", Value::Array(Arc::new(bad)), "list"); + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(v(&s, "A1"), Value::Error(CellError::Spill)); + assert!(s.doc().unwrap().wb.active().unwrap().spill.is_empty()); +} + +#[test] +fn py_formulas_over_the_size_limit_keep_only_their_value() { + let lits = |n: usize| { + let mut v = vec![format!("\"{}\"", "x".repeat(255)); n / 255]; + if !n.is_multiple_of(255) { + v.push(format!("\"{}\"", "x".repeat(n % 255))); + } + v.join("&") + }; + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + let cell = |text: String| Cell { value: Value::text("cached"), ..Cell::formula(gridcraft_model::Formula::new(&text)) }; + sh.set_cell(CellRef::new(0, 0), cell(format!("_xlfn._xlws.PY({},0)", lits(8_080)))); + sh.set_cell(CellRef::new(1, 0), cell(format!("_xlfn._xlws.PY({},0)", lits(8_079)))); + let xlsx = gridcraft_xlsx::write_xlsx(&wb).unwrap(); + let (wb, warnings) = crate::io::open_bytes("long.xlsx", &xlsx).unwrap(); + let sh = wb.sheet(0).unwrap(); + let over = sh.cell(CellRef::new(0, 0)).unwrap(); + assert!(over.py.is_none() && over.formula.is_none(), "8,080 characters of code is over the limit"); + assert_eq!(over.value, Value::text("cached")); + let within = sh.cell(CellRef::new(1, 0)).unwrap(); + assert_eq!(within.py.as_ref().map(|p| p.code.chars().count()), Some(8_079)); + assert!(warnings.iter().any(|w| w.contains("A1")), "{warnings:?}"); +} + +#[test] +fn a_dropped_python_result_takes_its_spill_with_it() { + // A1's code is over the limit and B1's result is malformed; each had spilled down to row 3, + // and C1's well-formed result keeps its spill. + let mut wb = Workbook::new(); + let sh = wb.sheet_mut(0).unwrap(); + let out = |value: Value| Some(PyOutput { value, type_name: "list".into(), preview: String::new(), convertible: true }); + let col = array(3, 1, vec![Value::Number(7.0), Value::Number(8.0), Value::Number(9.0)]); + let bad = Value::Array(Arc::new(Array { rows: 3, cols: 1, data: vec![Value::Number(7.0)] })); + for (at, code, value) in [("A1", "y".repeat(9_000), col.clone()), ("B1", "b".into(), bad), ("C1", "c".into(), col)] { + let at = CellRef::parse(at).unwrap(); + let py = PyCode { output: out(value), ..PyCode::new(&code) }; + sh.set_cell(at, Cell { value: Value::Number(7.0), formula: None, style: Default::default(), py: Some(Arc::new(py)) }); + let r = RangeRef::new(at, CellRef::new(2, at.col)); + sh.spill_ranges.insert(at, r); + for c in r.iter().skip(1) { + sh.spill.insert(c, Value::Number(8.0)); + } + } + let warnings = crate::io::enforce_python_limits(&mut wb); + assert!(warnings.iter().any(|w| w.contains("A1")), "{warnings:?}"); + let sh = wb.sheet(0).unwrap(); + for (anchor, below) in [("A1", "A2"), ("B1", "B3")] { + assert!(!sh.spill_ranges.contains_key(&CellRef::parse(anchor).unwrap()), "{anchor}'s spill range stays"); + assert_eq!(sh.value(CellRef::parse(below).unwrap()), Value::Empty, "{below} still shows {anchor}'s old spill"); + assert_eq!(sh.value(CellRef::parse(anchor).unwrap()), Value::Number(7.0), "{anchor} keeps its value"); + } + assert_eq!(sh.value(CellRef::parse("C2").unwrap()), Value::Number(8.0), "a sound result keeps its spill"); + // A save writes no value for the dropped spills. + let xml = sheet_xml(&gridcraft_xlsx::write_xlsx(&wb).unwrap()); + assert!(!xml.contains("r=\"A2\"") && !xml.contains("r=\"B3\""), "{xml}"); + assert!(xml.contains("r=\"C2\""), "{xml}"); +} + +#[test] +fn a_formula_entered_in_a_python_spill_takes_the_area_as_a_formula_would() { + // A1's cached array spills over A1:A3; =SEQUENCE(2) entered in A2 is not blocked by that old + // spill (the Python cell is laid out again in the same pass), so it spills and A1 is blocked. + let mut s = s(); + py(&mut s, "A1", "[1, 2, 3]"); + cache(&mut s, "A1", array(3, 1, vec![10.0.into(), 20.0.into(), 30.0.into()]), "list"); + s.execute("formulas.calculateNow", json!({})).unwrap(); + assert_eq!(v(&s, "A3"), Value::Number(30.0)); + s.execute("cell.set", json!({"cell": "A2", "input": "=SEQUENCE(2)"})).unwrap(); + assert_eq!((v(&s, "A2"), v(&s, "A3")), (Value::Number(1.0), Value::Number(2.0))); + assert_eq!(v(&s, "A1"), Value::Error(CellError::Spill)); + let wb = &s.doc().unwrap().wb; + let mut fresh = (**wb).clone(); + gridcraft_calc::Calc::new().recalc_all(&mut fresh); + for c in ["A1", "A2", "A3"] { + let c = CellRef::parse(c).unwrap(); + assert_eq!(wb.active().unwrap().value(c), fresh.active().unwrap().value(c), "{c}"); + } +} diff --git a/crates/engine/tests/it/command_sequence_fuzz.rs b/crates/engine/tests/it/command_sequence_fuzz.rs index 21a856c75..3c241995e 100644 --- a/crates/engine/tests/it/command_sequence_fuzz.rs +++ b/crates/engine/tests/it/command_sequence_fuzz.rs @@ -32,7 +32,7 @@ use std::collections::BTreeSet; use std::sync::Arc; -use gridcraft_core::{CellRef, MAX_COLS, MAX_ROWS, RangeRef, Value, col_to_letters}; +use gridcraft_core::{CellError, CellRef, MAX_COLS, MAX_ROWS, RangeRef, Value, col_to_letters}; use gridcraft_engine::calc::Calc; use gridcraft_engine::model::{CalcMode, Formula, Sheet, Workbook}; use gridcraft_engine::{EngineError, Session, command_specs, find_command, io}; @@ -433,7 +433,12 @@ fn value_diff(a: &Workbook, b: &Workbook, what: &str) -> Option { } for c in keys { let (va, vb) = (x.value(c), y.value(c)); - if !same_value(&va, &vb) { + // A file caches the Python errors as #VALUE! (they aren't error codes of the file + // format), in a Python cell and in the formulas that read one; reading it back runs + // nothing. + let python_error = matches!(va, Value::Error(CellError::Python | CellError::Busy | CellError::Timeout | CellError::Blocked)) + && vb == Value::Error(CellError::Value); + if !same_value(&va, &vb) && !python_error { let f = x.cell(c).and_then(|c| c.formula.as_ref()).map(|f| format!(" (formula ={})", f.text)).unwrap_or_default(); return Some(format!("{}!{c}{f}: {va:?} in memory, {vb:?} after {what}", x.name)); } @@ -785,6 +790,7 @@ const CURATED: &[(&str, usize)] = &[ ("formulas.defineName", 2), ("formulas.deleteName", 1), ("edit.replace", 1), + ("python.set", 3), ]; fn generate(rng: &mut Rng, s: &Session) -> Step { @@ -818,6 +824,25 @@ fn curated_params(rng: &mut Rng, wb: &Workbook, id: &str) -> Json { p } "range.setValues" => json!({"range": near_cell(rng).a1(), "values": values_2d(rng, wb)}), + "python.set" => { + // Code with xl() references (which follow structural edits), quotes and line breaks; + // sometimes long enough to meet the formula limit. + let mut code = match rng.below(5) { + 0 => format!("xl(\"{}\") + 1", near_range(rng).a1()), + 1 => format!("a = xl('{}')\nb = \"q\\\"uote\"\na", near_cell(rng).a1()), + 2 => "[[1, 2], [3, 4]]".to_string(), + 3 => String::new(), + _ => "\"".repeat(rng.int(1, 5000) as usize), + }; + if rng.chance(5) { + code.push_str(&"x".repeat(8_100)); + } + let mut p = json!({"cell": near_cell(rng).a1(), "code": code}); + if rng.chance(30) { + p["output"] = json!(if rng.chance(50) { "object" } else { "value" }); + } + p + } "range.fill" => json!({"range": near_range(rng).a1(), "input": input(rng, wb)}), "selection.set" => { let mut p = json!({"range": range(rng)}); diff --git a/crates/model/src/python.rs b/crates/model/src/python.rs index abd4c9414..c8a993915 100644 --- a/crates/model/src/python.rs +++ b/crates/model/src/python.rs @@ -1,14 +1,19 @@ //! Python cells: the code a cell holds, its last result, and the text forms around it (the -//! `PY(…)` formula files store it as, and the `xl("…")` references inside it). +//! `PY("…",0|1)` formula of the function's syntax, and the `xl("…")` references inside it). An +//! xlsx file keeps the code in its Python part instead (see the xlsx crate). use gridcraft_core::{CellError, Value}; use serde::{Deserialize, Serialize}; /// Excel's limit on the characters of one formula. A Python cell's code must fit in it once it is -/// encoded as a file formula ([`PyCode::to_file_formula`]): quotes are doubled and every 255 -/// characters add `"&"`, so the limit on the code itself depends on what the code holds. +/// written as the PY formula of the function's syntax ([`PyCode::to_file_formula`]): quotes are +/// doubled and every 255 characters add `"&"`, so the limit on the code itself depends on what the +/// code holds. pub const MAX_FORMULA_CHARS: usize = 8_192; +/// The most cells a Python result may have (a crafted file can claim any size). +pub const MAX_RESULT_CELLS: usize = 1_000_000; + /// The prefix files put before `PY(`, as Excel writes it ([MS-XLSX]: `_xlfn._xlws.PY`). pub const FILE_PREFIX: &str = "_xlfn._xlws."; @@ -66,13 +71,14 @@ impl PyCode { format!("PY({lits},{})", u8::from(self.object)) } - /// The formula a file stores the cell as: [`PyCode::to_formula`] with Excel's - /// `_xlfn._xlws.` prefix, so Excel reads it as its own Python function. + /// [`PyCode::to_formula`] with Excel's `_xlfn._xlws.` prefix: the string-literal form + /// [`PyCode::from_formula`] reads from files that hold it, and what the size limit measures. + /// An xlsx file names the code by index instead ([MS-XLSX] 2.3.10.2). pub fn to_file_formula(&self) -> String { format!("{FILE_PREFIX}{}", self.to_formula()) } - /// Whether the code fits in a file formula of at most [`MAX_FORMULA_CHARS`] characters. + /// Whether the code fits in a PY formula of at most [`MAX_FORMULA_CHARS`] characters. pub fn fits(&self) -> bool { // Code longer than the limit can't fit; this also bounds the work for hostile input. self.code.chars().nth(MAX_FORMULA_CHARS).is_none() && self.to_file_formula().chars().nth(MAX_FORMULA_CHARS).is_none() diff --git a/crates/xlsx/src/lib.rs b/crates/xlsx/src/lib.rs index 650b22820..b687c35dc 100644 --- a/crates/xlsx/src/lib.rs +++ b/crates/xlsx/src/lib.rs @@ -24,6 +24,7 @@ mod ods; mod package; mod passthrough; mod pivot; +mod python_part; mod read; mod richdata; mod sheet_read; diff --git a/crates/xlsx/src/python_part.rs b/crates/xlsx/src/python_part.rs new file mode 100644 index 000000000..11793f9fc --- /dev/null +++ b/crates/xlsx/src/python_part.rs @@ -0,0 +1,318 @@ +//! The Python part ([MS-XLSX] 2.1.23, 2.3.10): where a workbook keeps the code of its Python cells. +//! +//! A Python cell's formula is `_xlfn._xlws.PY(index, return_type, …)` ([MS-XLSX] 2.3.10.2): the +//! first argument is the zero-based index of a `pythonScript` element in this part, whose `code` +//! is the cell's Python code, and the second is 0 (an Excel value) or 1 (a Python object). Further +//! arguments are references the code names as `%Px%`, `x` being the argument's zero-based index +//! ([MS-XLSX] 2.3.10.4). The part's root is `python` ([MS-XLSX] 2.4.102, CT_Python 2.6.249), with +//! `pythonScripts` (2.6.253) holding the `pythonScript` elements (2.6.252). +//! +//! GridCraft writes one script per Python cell, in the order the cells are written, with the code +//! as it is (its `xl()` references stay text in the code, so no further arguments). Reading +//! resolves the index and puts each `%Px%` back as the text of argument `x`. A formula that names +//! no script the part has keeps only its cached value. Hostile input is bounded: the part's size, +//! the number of scripts, each script's length, and the code all cells take from the part +//! together. + +use gridcraft_formula::Expr; +use gridcraft_model::PyCode; +use gridcraft_model::python::{FILE_PREFIX, MAX_FORMULA_CHARS}; + +use crate::IoError; +use crate::read::Ctx; +use crate::xml::{self, El}; + +/// Namespace of the part. +pub const NS: &str = "http://schemas.microsoft.com/office/spreadsheetml/2023/python"; +/// Relationship type from the workbook part. +pub const REL: &str = "http://schemas.microsoft.com/office/2023/09/relationships/Python"; +/// Last segment of [`REL`], as [`crate::package::Rel::kind`] holds it. +const REL_KIND: &str = "Python"; +/// Content type of the part. +pub const CONTENT_TYPE: &str = "application/vnd.ms-excel.python+xml"; +/// Part name we write (the format names none; the relationship finds it). +pub const PART: &str = "xl/python.xml"; + +/// Largest Python part read. +const MAX_PART_BYTES: u64 = 64 * 1024 * 1024; +/// Most scripts read from the part. +const MAX_SCRIPTS: usize = 1_000_000; +/// Most characters kept of one script: one more than a cell's code may have, so a longer script +/// still fails [`PyCode::fits`] and its cell keeps only its value, with the usual warning. +const MAX_SCRIPT_CHARS: usize = MAX_FORMULA_CHARS + 1; +/// Most bytes of code all the cells of a workbook take from the part together. Several formulas +/// may name one script, so without it a small part could fill memory. +pub const MAX_CODE_BYTES: usize = 64 * 1024 * 1024; + +/// The scripts read from a workbook's Python part, and what the cells took from them. +#[derive(Default)] +pub struct ScriptsIn { + scripts: Vec, + /// Bytes of code the cells may still take ([`MAX_CODE_BYTES`] in all). + budget: usize, + /// Python cells that kept only their value: no such script, or over the budget. + missing: usize, +} + +impl ScriptsIn { + /// The warning for the Python cells that kept only their value, if any. + pub fn warning(&self) -> Option { + (self.missing > 0).then(|| { + format!( + "{} Python cell(s) name code the file's Python part doesn't hold (or holds too much of); they kept only their values", + self.missing + ) + }) + } +} + +/// Reads the workbook's Python part, if it has one. A damaged part is a warning: its cells keep +/// their values. +pub fn read(cx: &mut Ctx<'_>, rels: &[crate::package::Rel]) -> Result { + let mut out = ScriptsIn { budget: MAX_CODE_BYTES, ..ScriptsIn::default() }; + let Some(rel) = rels.iter().find(|r| !r.external && r.kind.eq_ignore_ascii_case(REL_KIND)) else { return Ok(out) }; + let bytes = match cx.pkg.read_limited(&rel.target, MAX_PART_BYTES) { + Ok(Some(b)) => b, + Ok(None) => { + cx.warn(format!("missing part {}", rel.target)); + return Ok(out); + } + Err(IoError::TooLarge(m)) => { + cx.warn(format!("the Python part is too large ({m}); Python cells kept only their values")); + return Ok(out); + } + Err(e) => return Err(e), + }; + let mut over = 0usize; + let parsed = xml::parse_streaming(&bytes, &["pythonScript"], &mut |el: El| { + if out.scripts.len() < MAX_SCRIPTS { + out.scripts.push(script_code(&el)); + } else { + over += 1; + } + Ok(()) + }); + match parsed { + Ok(root) if root.name != "python" => { + cx.warn(format!("the Python part has the root <{}>; Python cells kept only their values", root.name)); + out.scripts.clear(); + } + Ok(_) => {} + Err(e) => cx.warn(format!("the Python part is damaged ({e}); kept {} scripts", out.scripts.len())), + } + if over > 0 { + cx.warn(format!("the Python part holds more than {MAX_SCRIPTS} scripts; {over} were not read")); + } + Ok(out) +} + +/// A `pythonScript` element's code (ST_Xstring), with `\n` line breaks, cut after +/// [`MAX_SCRIPT_CHARS`] characters. +fn script_code(el: &El) -> String { + let text = el.child("code").map(|c| c.text.as_str()).unwrap_or_default(); + let text = match text.char_indices().nth(MAX_SCRIPT_CHARS) { + Some((at, _)) => text.get(..at).unwrap_or_default(), + None => text, + }; + gridcraft_model::python::normalize_code(&xml::decode_xstring(text)) +} + +/// `PY(index, 0|1, params…)`, as a file's formula holds it: the index, whether the return type is +/// a Python object, and the parameters. `None` for anything else (the code in a string literal +/// is [`PyCode::from_formula`]'s form). +fn py_call(expr: &Expr) -> Option<(usize, bool, &[Expr])> { + let Expr::Call(name, args) = expr else { return None }; + let base = name.trim_start_matches("_XLFN.").trim_start_matches("_XLWS."); + if !base.eq_ignore_ascii_case("PY") { + return None; + } + let index = match args.first()? { + Expr::Number(n) if *n >= 0.0 && n.fract() == 0.0 && *n < MAX_SCRIPTS as f64 => *n as usize, + _ => return None, + }; + let object = match args.get(1)? { + Expr::Number(n) if *n == 0.0 => false, + Expr::Number(n) if *n == 1.0 => true, + _ => return None, + }; + Some((index, object, args.get(2..).unwrap_or_default())) +} + +/// What a file formula is to Python. +#[derive(Debug, PartialEq)] +pub enum PyFormula { + /// Not a `PY(…)` call: an ordinary formula. + Not, + /// A Python cell with this code. + Code(PyCode), + /// A `PY(index, …)` call naming a script the part doesn't hold (or over the budget): the + /// cell keeps only its cached value, since no calculation can evaluate the formula. + Missing, +} + +/// What a file formula makes: a Python cell from the Python part when it names a script, or from +/// string literals ([`PyCode::from_formula`]). `formula` is the parsed formula, `raw` the +/// formula's own text, which keeps line breaks in literal code. +pub fn cell_code(scripts: &mut ScriptsIn, formula: &gridcraft_model::Formula, raw: &str) -> PyFormula { + // The cheap test first: most formulas are not `PY(` calls. + if !gridcraft_model::python::is_py_formula(&formula.text) { + return PyFormula::Not; + } + let expr = formula.expr_arc(); + let Some((index, object, params)) = expr.as_deref().and_then(py_call) else { + return PyCode::from_formula(raw).map_or(PyFormula::Not, PyFormula::Code); + }; + let code = scripts.scripts.get(index).map(|code| with_params(code, params)); + match code.filter(|c| c.len() <= scripts.budget) { + Some(code) => { + scripts.budget -= code.len(); + PyFormula::Code(PyCode { object, ..PyCode::new(&code) }) + } + None => { + scripts.missing += 1; + PyFormula::Missing + } + } +} + +/// `code` with every `%Px%` (`x` at least 2, naming one of `params`) replaced by the parameter's +/// text. Stops growing past [`MAX_SCRIPT_CHARS`] characters: such code can't be a cell's. +fn with_params(code: &str, params: &[Expr]) -> String { + if params.is_empty() || !code.contains("%P") { + return code.to_string(); + } + let texts: Vec = params.iter().map(gridcraft_formula::print).collect(); + let mut out = String::with_capacity(code.len()); + let mut chars = 0usize; + let mut push = |out: &mut String, piece: &str| { + out.push_str(piece); + chars = chars.saturating_add(piece.chars().count()); + chars > MAX_SCRIPT_CHARS + }; + let mut rest = code; + while let Some(at) = rest.find("%P") { + if push(&mut out, rest.get(..at).unwrap_or_default()) { + return out; + } + let after = rest.get(at + 2..).unwrap_or_default(); + let digits = after.bytes().take_while(u8::is_ascii_digit).count(); + let param = (1..=6) + .contains(&digits) + .then(|| after.get(..digits).and_then(|d| d.parse::().ok())) + .flatten() + .filter(|_| after.as_bytes().get(digits) == Some(&b'%')) + .and_then(|x| x.checked_sub(2)) + .and_then(|x| texts.get(x)); + let full = match param { + Some(text) => { + rest = after.get(digits + 1..).unwrap_or_default(); + push(&mut out, text) + } + None => { + rest = after; + push(&mut out, "%P") + } + }; + if full { + return out; + } + } + out.push_str(rest); + out +} + +/// The formula a Python cell is saved as: `_xlfn._xlws.PY(index,0|1)`, its code being script +/// `index` of the part ([`write`]). +pub fn file_formula(index: usize, object: bool) -> String { + format!("{FILE_PREFIX}PY({index},{})", u8::from(object)) +} + +/// The Python part holding `scripts`, in order. Only written when there is at least one: the +/// format requires one `pythonScript` in `pythonScripts`. +pub fn write(scripts: &[String]) -> String { + let mut xml = format!("{}", crate::write::XML_DECL); + for code in scripts { + xml.push_str(""); + xml.push_str(&xml::esc(&xml::encode_xstring(code))); + xml.push_str(""); + } + xml.push_str(""); + xml +} + +#[cfg(test)] +mod tests { + use super::*; + + fn formula(text: &str) -> gridcraft_model::Formula { + gridcraft_model::Formula::new(&crate::fmla::from_file(text)) + } + + fn scripts(list: &[&str]) -> ScriptsIn { + ScriptsIn { scripts: list.iter().map(|s| s.to_string()).collect(), budget: MAX_CODE_BYTES, missing: 0 } + } + + #[test] + fn reads_the_index_form_and_parameters() { + let mut s = scripts(&["1 + 1", "df = xl(\"%P2%\", headers=True)\nxl('%P3%') + %P9% + %P1%"]); + let code = |s: &mut ScriptsIn, text: &str| match cell_code(s, &formula(text), text) { + PyFormula::Code(py) => Some(py), + _ => None, + }; + let a = code(&mut s, "_xlfn._xlws.PY(0,1)").unwrap(); + assert_eq!((a.code.as_str(), a.object), ("1 + 1", true)); + let b = code(&mut s, "_xlfn._xlws.PY(1,0,A1:B3,Sheet2!$C$4)").unwrap(); + assert_eq!(b.code, "df = xl(\"A1:B3\", headers=True)\nxl('Sheet2!$C$4') + %P9% + %P1%"); + assert!(!b.object); + // The literal form still reads (other applications, older files). + assert_eq!(code(&mut s, "_xlfn._xlws.PY(\"x = 1\",0)").unwrap().code, "x = 1"); + // A script the part doesn't hold: the cell keeps its value, and is counted for the warning. + let gone = "_xlfn._xlws.PY(7,0)"; + assert_eq!(cell_code(&mut s, &formula(gone), gone), PyFormula::Missing); + assert!(s.warning().is_some()); + for not in ["_xlfn._xlws.PY(-1,0)", "_xlfn._xlws.PY(0.5,0)", "_xlfn._xlws.PY(0,2)", "PY(A1,0)", "SUM(0,1)", "PYTHON_TYPE(A1)"] { + assert_eq!(cell_code(&mut s, &formula(not), not), PyFormula::Not, "{not}"); + } + } + + #[test] + fn hostile_scripts_are_bounded() { + // Many formulas naming one script take from one budget. + let code = "x".repeat(1_000); + let mut s = ScriptsIn { scripts: vec![code], budget: 2_500, missing: 0 }; + let f = formula("_xlfn._xlws.PY(0,0)"); + let taken = (0..5).filter(|_| matches!(cell_code(&mut s, &f, ""), PyFormula::Code(_))).count(); + assert_eq!(taken, 2); + assert_eq!(s.missing, 3); + // A parameter repeated many times stops growing the code just past a cell's limit. + let many = "%P2%".repeat(5_000); + let out = with_params(&many, &[Expr::Name("A_VERY_LONG_NAME_FOR_A_RANGE".into())]); + assert!((MAX_SCRIPT_CHARS..MAX_SCRIPT_CHARS + 40).contains(&out.chars().count()), "{}", out.len()); + assert!(!PyCode::new(&out).fits()); + // Odd tokens stay as they are. + assert_eq!(with_params("%P%P2%%P99999999%%P", &[Expr::Number(1.0)]), "%P1%P99999999%%P"); + // A long script is cut just past the limit, so it fails the cell's check. + let el = El { + name: "pythonScript".into(), + children: vec![El { name: "code".into(), text: "y".repeat(20_000), ..El::default() }], + ..El::default() + }; + assert_eq!(script_code(&el).chars().count(), MAX_SCRIPT_CHARS); + } + + #[test] + fn written_scripts_read_back() { + let codes = vec!["a = 1\nb = \"two\" & 'three' ".to_string(), "_x0041_ \u{1}".to_string(), String::new()]; + let xml = write(&codes); + assert!(xml.contains(NS) && xml.contains("a = 1\nb = \"two\" & 'three' <tag>"), "{xml}"); + let mut read = Vec::new(); + let root = xml::parse_streaming(xml.as_bytes(), &["pythonScript"], &mut |el| { + read.push(script_code(&el)); + Ok(()) + }) + .unwrap_or_default(); + assert_eq!(root.name, "python"); + assert_eq!(read, codes); + assert_eq!(file_formula(3, true), "_xlfn._xlws.PY(3,1)"); + } +} diff --git a/crates/xlsx/src/read.rs b/crates/xlsx/src/read.rs index 55ef6757d..d7b9240e2 100644 --- a/crates/xlsx/src/read.rs +++ b/crates/xlsx/src/read.rs @@ -32,6 +32,8 @@ pub struct Ctx<'a> { pub cell_pictures: Vec>>, /// x14 conditional-format ids of the current sheet's data bars, with the index of the rule in `cond_formats`. pub cf_ext_ids: Vec<(String, usize)>, + /// The code of the workbook's Python cells, from its Python part. + pub python: crate::python_part::ScriptsIn, } impl Ctx<'_> { @@ -113,6 +115,7 @@ pub fn read_xlsx(bytes: &[u8]) -> Result<(Workbook, ReadReport), IoError> { num_fmts: Default::default(), cell_pictures: vec![], cf_ext_ids: vec![], + python: Default::default(), }; let root_rels = cx.pkg.rels("")?; let wb_part = find_rel(&root_rels, "officeDocument").map(|r| r.target.clone()).unwrap_or_else(|| "xl/workbook.xml".into()); @@ -140,6 +143,8 @@ pub fn read_xlsx(bytes: &[u8]) -> Result<(Workbook, ReadReport), IoError> { vec![] } }; + // The code of Python cells, which their formulas name by index. + cx.python = crate::python_part::read(&mut cx, &wb_rels)?; let mut wb = Workbook::new(); wb.sheets.clear(); @@ -394,6 +399,9 @@ pub fn read_xlsx(bytes: &[u8]) -> Result<(Workbook, ReadReport), IoError> { } } wb.preserved = kept.workbook(&mut cx, &wb_part, &wb_xml, &file_to_model); + if let Some(w) = cx.python.warning() { + cx.warn(w); + } // Calls of LAMBDA names are spelled like the names (the parser upper-cases them). wb.apply_name_call_case(); wb.styles = cx.styles; diff --git a/crates/xlsx/src/sheet_read.rs b/crates/xlsx/src/sheet_read.rs index bec67bd1a..5945a9669 100644 --- a/crates/xlsx/src/sheet_read.rs +++ b/crates/xlsx/src/sheet_read.rs @@ -41,16 +41,17 @@ struct RowState { rows: u64, } -/// Whether `pos` lies in a known dynamic range that isn't its anchor: the cell's cached value -/// gives way to the formula that will fill it. Charges every scanned range against one work -/// budget, then gives up consulting the ranges at all: the worst kind of file may then import -/// a few cached spill values, but it never stalls the whole import. -fn in_dynamic(st: &mut RowState, pos: CellRef) -> bool { +/// The known dynamic range `pos` lies in, when it isn't the range's anchor: the cell's cached +/// value gives way to the formula that will fill it (or, for a Python cell, becomes its +/// result). Charges every scanned range against one work budget, then gives up consulting the +/// ranges at all: the worst kind of file may then import a few cached spill values, but it +/// never stalls the whole import. +fn dynamic_range(st: &mut RowState, pos: CellRef) -> Option { if st.dynamic_checks > MAX_DYNAMIC_CHECKS || st.dynamic.is_empty() { - return false; + return None; } let ranges = st.dynamic.len(); - let hit = st.dynamic.iter().any(|r| r.contains(pos) && r.start != pos); + let hit = st.dynamic.iter().find(|r| r.contains(pos) && r.start != pos).copied(); st.dynamic_checks = st.dynamic_checks.saturating_add(ranges as u64); hit } @@ -515,8 +516,34 @@ fn read_cell(cx: &mut Ctx<'_>, sheet: &mut Sheet, st: &mut RowState, c: &El, pos } let dynamic = c.attr("cm").is_some(); let formula = c.child("f").and_then(|f| read_formula(cx, st, f, pos, dynamic)); - let value = if formula.is_none() && in_dynamic(st, pos) { Value::Empty } else { value }; - sheet.cells.set(pos, Cell { value, formula: formula.map(Arc::new), style, py: None }); + // A `PY(index,0|1)` formula is a Python cell whose code is a script of the Python part. The + // older `PY("…",0|1)` form is read from the formula's own text: the formula parser would fold + // line breaks in the code. A formula naming a script the part lacks keeps only its value. + let py = match (&formula, c.child("f")) { + (Some(fm), Some(f)) => crate::python_part::cell_code(&mut cx.python, fm, &f.text), + _ => crate::python_part::PyFormula::Not, + }; + let (formula, py) = match py { + crate::python_part::PyFormula::Not => (formula, None), + crate::python_part::PyFormula::Code(py) => (None, Some(Arc::new(py))), + crate::python_part::PyFormula::Missing => (None, None), + }; + let spilled_from = if formula.is_none() && py.is_none() { dynamic_range(st, pos) } else { None }; + let value = match spilled_from { + // A Python cell's spilled values are kept: they are its result until it runs again. A + // range larger than a result may be is a crafted file's: its values are dropped, like a + // formula's spilled values, and no spill is recorded. + Some(r) if r.count() <= gridcraft_model::python::MAX_RESULT_CELLS as u64 && sheet.cells.get(r.start).is_some_and(|a| a.py.is_some()) => { + if !value.is_empty() { + sheet.spill.insert(pos, value); + } + sheet.spill_ranges.insert(r.start, r); + Value::Empty + } + Some(_) => Value::Empty, + None => value, + }; + sheet.cells.set(pos, Cell { value, formula: formula.map(Arc::new), style, py }); } fn read_formula(cx: &mut Ctx<'_>, st: &mut RowState, f: &El, pos: CellRef, dynamic: bool) -> Option { diff --git a/crates/xlsx/src/sheet_write.rs b/crates/xlsx/src/sheet_write.rs index 9251edaa4..5203adc1b 100644 --- a/crates/xlsx/src/sheet_write.rs +++ b/crates/xlsx/src/sheet_write.rs @@ -119,7 +119,9 @@ pub fn write_sheet(wb: &Workbook, si: usize, selected: bool, out: &mut Out) -> ( let dynamic: BTreeMap = sheet .spill_ranges .iter() - .filter(|(a, r)| !r.is_single() && sheet.cells.get(**a).is_some_and(|c| c.formula.as_ref().is_some_and(|f| f.array.is_none()))) + .filter(|(a, r)| { + !r.is_single() && sheet.cells.get(**a).is_some_and(|c| c.py.is_some() || c.formula.as_ref().is_some_and(|f| f.array.is_none())) + }) .map(|(a, r)| (*a, *r)) .collect(); let mut extra: BTreeMap = BTreeMap::new(); @@ -686,6 +688,17 @@ fn write_cell( return; } let formula = if ov.is_some() { None } else { cell.and_then(|x| x.formula.as_ref()) }; + // A Python cell is saved as [MS-XLSX] 2.3.10 specifies, `_xlfn._xlws.PY(index,0|1)` with its + // code as script `index` of the Python part, and with its cached value; an application without + // Python shows the value (and #NAME? if it recalculates). + let py_text = if ov.is_some() || formula.is_some() { + None + } else { + cell.and_then(|x| x.py.as_ref()).map(|p| { + out.python_scripts.push(p.code.clone()); + crate::python_part::file_formula(out.python_scripts.len().saturating_sub(1), p.object) + }) + }; let value: Value = match (ov, extra) { (Some(t), _) => Value::text(t.as_str()), (None, Some(v)) => v.clone(), @@ -709,11 +722,12 @@ fn write_cell( let own = formula.filter(|f| f.dynamic && f.array.is_none()).map(|_| RangeRef::cell(c)); let dynamic = dynamic.or(own.as_ref()); // A non-spilling formula may still need array evaluation to avoid implicit intersection. - if dynamic.is_some() && formula.is_some_and(|f| f.array.is_none()) { + if dynamic.is_some() && (formula.is_some_and(|f| f.array.is_none()) || py_text.is_some()) { s.push_str(" cm=\"1\""); out.dynamic_arrays = true; } - if let Some(f) = formula { + let text = formula.map(|f| esc(&crate::fmla::to_file(f))).or_else(|| py_text.map(|t| esc(&t))); + if let Some(text) = text { let t = match &value { Value::Text(_) => " t=\"str\"", Value::Bool(_) => " t=\"b\"", @@ -722,8 +736,7 @@ fn write_cell( }; s.push_str(t); s.push('>'); - let text = esc(&crate::fmla::to_file(f)); - match (f.array, dynamic) { + match (formula.and_then(|f| f.array), dynamic) { (Some(r), _) => { let _ = write!(s, "{text}", r.a1()); } diff --git a/crates/xlsx/src/write.rs b/crates/xlsx/src/write.rs index b96df35c7..41ab3e8a3 100644 --- a/crates/xlsx/src/write.rs +++ b/crates/xlsx/src/write.rs @@ -128,6 +128,8 @@ pub struct Out { pub cell_pictures: crate::richdata::PicturesOut, /// Kept (pass-through) parts written so far. pub kept: crate::passthrough::KeptOut, + /// The code of the Python cells written so far, in order: script `i` of the Python part. + pub python_scripts: Vec, } impl Out { @@ -237,6 +239,7 @@ pub fn write_xlsx_named(wb: &Workbook, kind: XlsxKind, book_name: Option<&str>) needs_calc: false, cell_pictures: Default::default(), kept: Default::default(), + python_scripts: vec![], }; out.kept.reserve_vml_blocks(wb); out.default_type("rels", "application/vnd.openxmlformats-package.relationships+xml"); @@ -399,6 +402,12 @@ pub fn write_xlsx_named(wb: &Workbook, kind: XlsxKind, book_name: Option<&str>) pivots.finish(wb, &mut out, &|code: &str| crate::tables::builtin_id(code).or_else(|| fmt_ids.get(code).copied())); out.part("xl/styles.xml", Some("application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"), styles.into_bytes()); wb_rels.add("styles", "styles.xml"); + if !out.python_scripts.is_empty() { + use crate::python_part as py; + let part = py::write(&out.python_scripts); + out.part(py::PART, Some(py::CONTENT_TYPE), part.into_bytes()); + wb_rels.add_uri(py::REL, "python.xml"); + } if wb.styles.iter().any(|(_, st)| st.num_fmt.as_str() == "checkbox") { use crate::feature_bags as fpb; out.part(fpb::PART, Some(fpb::CONTENT_TYPE), fpb::checkbox_part().into_bytes());