Compare commits

..
Author SHA1 Message Date
Dylan M. Taylor 1e4bfecdf3 calc: check the dependency closure against a walk of every cell
Random graphs of single-cell, narrow and wide range reads (some removed or unsettled), random seeds and changed areas: the closure with shared taken records and area lookups equals the one listing every dependent of every changed cell.
2026-11-02 22:35:55 -05:00
Dylan M. Taylor 992c1e35f0 calc: list each dependent once when a change spreads 2026-11-02 20:29:56 -05:00
3 changed files with 283 additions and 145 deletions
+178 -14
View File
@@ -24,7 +24,7 @@ mod pass_host;
#[cfg(test)]
use graph::Intervals;
pub use graph::{FxHasher, Graph, Key};
use graph::{HashMap, HashSet, Node};
use graph::{HashMap, HashSet, Node, Taken};
use pass_host::{PassHost, spills_as_before};
/// Calculation state kept beside a workbook.
@@ -292,17 +292,16 @@ impl Calc {
}
}
}
// Spill areas of changed anchors also change.
// Spill areas of changed anchors also change. They stay rectangles: one seed per spilled
// cell listed the formulas reading the area once per cell.
let mut areas: Vec<(usize, RangeRef)> = cleared;
for &k in changed {
if let Some(r) = wb.sheet(k.0).and_then(|s| s.spill_ranges.get(&k.1)) {
seeds.extend(r.iter().map(|c| (k.0, c)));
areas.push((k.0, *r));
}
}
for (si, r) in &cleared {
seeds.extend(r.iter().map(|c| (*si, c)));
}
let t0 = prof_now();
let dirty = self.dirty_closure(wb, &seeds);
let dirty = self.dirty_closure(wb, &seeds, &areas);
let t1 = prof_now();
let n = dirty.len();
self.run(wb, dirty, false);
@@ -311,16 +310,39 @@ impl Calc {
}
}
fn dirty_closure(&self, _wb: &Workbook, seeds: &[Key]) -> Vec<Key> {
/// The formulas to recalculate after `seeds` and every cell of `areas` changed: the seeds
/// that are formulas, and everything that reads them, directly or through other formulas.
fn dirty_closure(&self, wb: &Workbook, seeds: &[Key], areas: &[(usize, RangeRef)]) -> Vec<Key> {
let mut seen: HashSet<Key> = HashSet::default();
let mut queue: VecDeque<Key> = VecDeque::new();
// Each range-index entry is listed once in the whole walk: many changed cells under one
// broad reference (or many formulas inside one) would otherwise list every formula reading
// it once per cell, which squared the work.
let mut taken = Taken::default();
let mut deps = Vec::new();
for k in seeds {
if self.graph.nodes.contains_key(k) && seen.insert(*k) {
queue.push_back(*k);
}
let mut deps = Vec::new();
self.graph.dependents(k.0, k.1, &mut deps);
for d in deps {
deps.clear();
self.graph.dependents_untaken(k.0, k.1, &mut taken, &mut deps);
for d in deps.drain(..) {
if seen.insert(d) {
queue.push_back(d);
}
}
}
for &(si, r) in areas {
if let Some(sh) = wb.sheet(si) {
for (c, _) in sh.cells.iter_range(r) {
if self.graph.nodes.contains_key(&(si, c)) && seen.insert((si, c)) {
queue.push_back((si, c));
}
}
}
deps.clear();
self.graph.dependents_of_range(si, r, &mut deps);
for d in deps.drain(..) {
if seen.insert(d) {
queue.push_back(d);
}
@@ -331,10 +353,9 @@ impl Calc {
queue.push_back(*k);
}
}
let mut deps = Vec::new();
while let Some(k) = queue.pop_front() {
deps.clear();
self.graph.dependents(k.0, k.1, &mut deps);
self.graph.dependents_untaken(k.0, k.1, &mut taken, &mut deps);
for d in deps.drain(..) {
if seen.insert(d) {
queue.push_back(d);
@@ -411,7 +432,7 @@ impl Calc {
if spill_changes.is_empty() {
break;
}
dirty = self.dirty_closure(wb, &spill_changes);
dirty = self.dirty_closure(wb, &spill_changes, &[]);
}
// Loops among formulas the run did not evaluate stay listed while they are still formulas.
// Iterative calculation reports none.
@@ -691,6 +712,149 @@ pub(crate) mod tests {
assert!(visited < 64, "visited {visited} nodes for no match");
}
#[test]
fn untaken_queries_list_each_interval_once_in_all() {
// 1,000 ranges over every row of 5,000: point queries at every row used to list all 1,000
// each time (5 million matches); sharing `taken`, each range is listed once in all.
let n = 5_000u32;
let mut iv: Intervals<u32> = Intervals::default();
for t in 0..1_000 {
iv.insert(0, n, t);
}
for r in 0..n {
iv.insert(r, r, 10_000 + r);
}
iv.settle();
let mut taken = Vec::new();
let mut listed = Vec::new();
for row in 0..=n + 3 {
iv.query_untaken(row, row, &mut taken, &mut |t| {
listed.push(*t);
true
});
}
listed.sort_unstable();
let want: Vec<u32> = (0..1_000).chain(10_000..10_000 + n).collect();
assert_eq!(listed, want);
// An entry the caller declines is offered again.
let mut taken = Vec::new();
let mut offered = 0;
for _ in 0..3 {
iv.query_untaken(7, 7, &mut taken, &mut |t| {
if *t == 10_007 {
offered += 1;
}
*t != 10_007
});
}
assert_eq!(offered, 3);
}
#[test]
fn dirty_closure_matches_a_walk_of_every_cell() {
// Random formulas over single cells, narrow ranges and wide (over 64 columns) ranges, some
// removed or left unsettled: the closure equals the one found by listing every dependent
// of every changed cell, area cells included, as before `taken` and areas existed.
use super::{HashSet, Node};
use crate::eval::Area;
use std::collections::VecDeque;
let mut s = 0x2545_F491_4F6C_DD1D;
for case in 0..40 {
let mut wb = Workbook::new();
let mut calc = Calc::new();
let pick = |s: &mut u64, n: u64| (rng(s) % n) as u32;
let mut keys: Vec<Key> = Vec::new();
for _ in 0..300 {
let k = (0, CellRef::new(pick(&mut s, 120), pick(&mut s, 150)));
let mut areas = Vec::new();
for _ in 0..1 + pick(&mut s, 3) {
let start = CellRef::new(pick(&mut s, 120), pick(&mut s, 150));
let end = match pick(&mut s, 3) {
0 => start,
1 => CellRef::new(start.row + pick(&mut s, 40), start.col + pick(&mut s, 5)),
_ => CellRef::new(start.row + pick(&mut s, 20), start.col + 65 + pick(&mut s, 30)),
};
areas.push(Area { sheet: 0, range: RangeRef::new(start, end) });
}
if let Some(sh) = wb.sheet_mut(0) {
sh.cells.set(k.1, Cell::formula(Formula::new("=1")));
}
calc.graph.insert(k, Node { areas, dynamic: false });
keys.push(k);
if rng(&mut s).is_multiple_of(7) {
calc.graph.settle();
}
}
for _ in 0..30 {
let k = keys[(rng(&mut s) % keys.len() as u64) as usize];
calc.graph.remove(k);
if let Some(sh) = wb.sheet_mut(0) {
sh.cells.remove(k.1);
}
}
if case % 2 == 0 {
calc.graph.settle();
}
let seeds: Vec<Key> = (0..1 + pick(&mut s, 25)).map(|_| (0, CellRef::new(pick(&mut s, 140), pick(&mut s, 220)))).collect();
let areas: Vec<(usize, RangeRef)> = (0..pick(&mut s, 3))
.map(|_| {
let start = CellRef::new(pick(&mut s, 120), pick(&mut s, 150));
(0, RangeRef::new(start, CellRef::new(start.row + pick(&mut s, 30), start.col + pick(&mut s, 80))))
})
.collect();
let got: HashSet<Key> = calc.dirty_closure(&wb, &seeds, &areas).into_iter().collect();
// The walk before: every area cell is a seed, and every cell lists all its dependents.
let mut all = seeds.clone();
for (si, r) in &areas {
all.extend(r.iter().map(|c| (*si, c)));
}
let mut want: HashSet<Key> = HashSet::default();
let mut queue: VecDeque<Key> = VecDeque::new();
let mut deps = Vec::new();
for k in &all {
if calc.graph.nodes.contains_key(k) && want.insert(*k) {
queue.push_back(*k);
}
deps.clear();
calc.graph.dependents(k.0, k.1, &mut deps);
queue.extend(deps.iter().copied().filter(|d| want.insert(*d)));
}
while let Some(k) = queue.pop_front() {
deps.clear();
calc.graph.dependents(k.0, k.1, &mut deps);
queue.extend(deps.iter().copied().filter(|d| want.insert(*d)));
}
assert_eq!(got, want, "case {case}");
}
}
#[test]
fn clearing_a_spill_recalculates_formulas_reading_it_once_each() {
// 300 formulas read the area an anchor spills into; clearing the anchor changes all of it
// and every one of them is recalculated (they were listed once per spilled cell).
let mut wb = Workbook::new();
let mut calc = Calc::new();
let set = |wb: &mut Workbook, calc: &mut Calc, at: CellRef, input: &str| {
if let Some(sh) = wb.sheet_mut(0) {
if input.is_empty() {
sh.cells.remove(at);
} else {
sh.cells.set(at, Cell::formula(Formula::new(input)));
}
}
calc.cells_changed(wb, &[(0, at)]);
};
set(&mut wb, &mut calc, CellRef::new(0, 0), "=SEQUENCE(2000)");
for r in 0..300 {
set(&mut wb, &mut calc, CellRef::new(r, 2), "=COUNT(A1:A2000)");
}
assert_eq!(wb.sheet(0).map(|s| s.value(CellRef::new(299, 2))), Some(Value::Number(2000.0)));
set(&mut wb, &mut calc, CellRef::new(0, 0), "");
for r in 0..300 {
assert_eq!(wb.sheet(0).map(|s| s.value(CellRef::new(r, 2))), Some(Value::Number(0.0)), "row {r}");
}
}
/// `rows` rows of six numbers (E:J) and `=SUM(E{r}:J{r})` in K.
fn sum_rows(rows: u32) -> Workbook {
let mut wb = Workbook::new();
+81
View File
@@ -151,6 +151,54 @@ impl<T: Copy + Ord> Intervals<T> {
self.walk(1, 0, self.max_end.len() / 2, p, a, f)
}
/// [`Intervals::query`] that skips entries an earlier call with the same `taken` already
/// took (and whole subtrees of them), so that a series of queries lists each interval once in
/// all. `f` answers whether it took the entry; one it declines is offered again later.
/// `taken` belongs to this index as it is now; start it empty. Pending insertions (a few at
/// most after [`Intervals::settle`]) are offered every time.
pub(super) fn query_untaken(&self, a: u32, b: u32, taken: &mut Vec<bool>, f: &mut dyn FnMut(&T) -> bool) {
for it in &self.pending {
if it.0 <= b && it.1 >= a {
f(&it.2);
}
}
let p = self.items.partition_point(|it| it.0 <= b);
if p == 0 {
return;
}
if taken.len() != self.max_end.len() {
*taken = vec![false; self.max_end.len()];
}
self.walk_untaken(1, 0, self.max_end.len() / 2, p, a, taken, f);
}
/// Returns whether everything under `node` has now been reported (or is dead).
#[allow(clippy::too_many_arguments)]
fn walk_untaken(&self, node: usize, lo: usize, hi: usize, p: usize, a: u32, taken: &mut [bool], f: &mut dyn FnMut(&T) -> bool) -> bool {
if taken.get(node).copied().unwrap_or(true) {
return true;
}
if lo >= p || self.max_end.get(node).is_none_or(|m| *m < a) {
return false;
}
let done = if hi - lo <= 1 {
match (self.items.get(lo), self.dead.get(lo)) {
(Some(it), Some(false)) if it.1 >= a => f(&it.2),
(Some(_), Some(true)) => true,
_ => false,
}
} else {
let mid = lo + (hi - lo) / 2;
let left = self.walk_untaken(2 * node, lo, mid, p, a, taken, f);
let right = self.walk_untaken(2 * node + 1, mid, hi, p, a, taken, f);
left && right
};
if done && let Some(slot) = taken.get_mut(node) {
*slot = true;
}
done
}
fn walk(&self, node: usize, lo: usize, hi: usize, p: usize, a: u32, f: &mut dyn FnMut(&T)) -> usize {
if lo >= p || self.max_end.get(node).is_none_or(|m| *m < a) {
return 1;
@@ -168,6 +216,13 @@ impl<T: Copy + Ord> Intervals<T> {
}
}
/// Range-index entries one dependency walk has already listed (see [`Graph::dependents_untaken`]).
#[derive(Default)]
pub(super) struct Taken {
cols: HashMap<(usize, u32), Vec<bool>>,
wide: HashMap<usize, Vec<bool>>,
}
#[derive(Clone, Debug, Default)]
pub struct Graph {
pub(super) nodes: HashMap<Key, Node>,
@@ -302,6 +357,32 @@ impl Graph {
}
}
/// [`Graph::dependents`] leaving out the formulas whose range reads an earlier call with the
/// same `taken` already listed. A walk over many changed cells that a few broad ranges cover
/// then lists those formulas once, instead of once per cell (which squared the work). The
/// graph must not change while `taken` is in use.
pub(super) fn dependents_untaken(&self, sheet: usize, c: CellRef, taken: &mut Taken, out: &mut Vec<Key>) {
if let Some(v) = self.cell_deps.get(&(sheet, c)) {
out.extend(v.iter().copied());
}
if let Some(v) = self.col_deps.get(&(sheet, c.col)) {
v.query_untaken(c.row, c.row, taken.cols.entry((sheet, c.col)).or_default(), &mut |k| {
out.push(*k);
true
});
}
if let Some(v) = self.wide_deps.get(&sheet) {
// A wide entry is taken once it covers the cell's column too.
v.query_untaken(c.row, c.row, taken.wide.entry(sheet).or_default(), &mut |(c0, c1, k)| {
let hit = *c0 <= c.col && c.col <= *c1;
if hit {
out.push(*k);
}
hit
});
}
}
/// Formulas that read any cell of `range` (a formula can be listed more than once).
pub fn dependents_of_range(&self, sheet: usize, range: RangeRef, out: &mut Vec<Key>) {
if range.count() <= 4096 {
+24 -131
View File
@@ -68,70 +68,26 @@ const SCANS: &[&str] = &["VLOOKUP", "HLOOKUP", "LOOKUP", "MATCH", "XLOOKUP", "XM
/// `XMATCH(SEQUENCE(1048576), -SEQUENCE(1048576))` runs for hours and hangs opening the file.
/// Past the budget the call is `#NUM!`, as MMULT is past 2e9 multiply-adds. 5e8 keeps the worst
/// case to a few seconds and still admits 5,000 needles over a 100,000-row table.
///
/// The same budget covers all the work of a lifted call added up across its elements: criteria
/// aggregates rescanning their ranges, period loops, and array results built per element.
const LIFTED_SCAN_BUDGET: usize = 500_000_000;
/// Criteria aggregates: every call tests each cell of its ranges against its criteria, so a call
/// lifted over `n` criteria does `n` x (range cells) work, like the [`SCANS`] lookups.
const CRITERIA: &[&str] = &["COUNTIF", "COUNTIFS", "SUMIF", "SUMIFS", "AVERAGEIF", "AVERAGEIFS", "MAXIFS", "MINIFS"];
/// Cumulative TVM functions that step through every period from `start_period` (argument 3) to
/// `end_period` (argument 4) on each call.
const PERIOD_LOOPS: &[&str] = &["CUMIPMT", "CUMPRINC"];
/// The work every lifted call of `spec` repeats over its non-lifted arguments: the longest vector
/// a [`SCANS`] lookup searches, or every cell a [`CRITERIA`] aggregate tests. 0 for other
/// functions.
/// The length of the longest vector a lifted call of `spec` searches per needle: 0 unless `spec`
/// is one of the [`SCANS`] lookups and a non-lifted argument is an array.
fn scan_len(spec: &FnSpec, args: &[Arg]) -> usize {
let scans = SCANS.contains(&spec.name);
if !scans && !CRITERIA.contains(&spec.name) {
if !SCANS.contains(&spec.name) {
return 0;
}
let lens = args.iter().enumerate().filter(|(i, _)| !spec.is_scalar_arg(*i)).filter_map(|(_, a)| match &a.value {
Value::Array(arr) => Some(if scans { arr.rows.max(arr.cols) } else { arr.data.len() }),
_ => None,
});
if scans { lens.max().unwrap_or(0) } else { lens.fold(0, usize::saturating_add) }
}
/// The work of one lifted call that depends on its own (lifted) scalar arguments: the number of
/// periods a [`PERIOD_LOOPS`] function steps through. 0 for other functions.
fn call_cost(spec: &FnSpec, cell_args: &[Arg]) -> usize {
if !PERIOD_LOOPS.contains(&spec.name) {
return 0;
}
let period = |i: usize| match cell_args.get(i).map(|a| &a.value) {
Some(Value::Number(n)) if n.is_finite() => Some(n.trunc()),
_ => None,
};
match (period(3), period(4)) {
// The function itself refuses spans past 1e7 periods; anything larger costs the cap.
(Some(start), Some(end)) if end >= start => ((end - start) + 1.0).min(1e8) as usize,
_ => 0,
}
args.iter()
.enumerate()
.filter(|(i, _)| !spec.is_scalar_arg(*i))
.filter_map(|(_, a)| match &a.value {
Value::Array(arr) => Some(arr.rows.max(arr.cols)),
_ => None,
})
.max()
.unwrap_or(0)
}
pub(crate) fn call_lifted(spec: &FnSpec, args: &[Arg], ctx: &mut dyn Ctx) -> Value {
call_lifted_within(spec, args, ctx, LIFTED_SCAN_BUDGET)
}
/// Sets the lifted arguments of `cell_args` to their elements at (`r`, `c`).
fn element_args(spec: &FnSpec, args: &[Arg], r: usize, c: usize, cell_args: &mut [Arg]) {
for (i, a) in args.iter().enumerate() {
if spec.is_scalar_arg(i)
&& let Value::Array(arr) = &a.value
&& let Some(slot) = cell_args.get_mut(i)
{
let v = arr.get_broadcast(r, c);
slot.value = if let Value::Array(inner) = &v { inner.data.first().cloned().unwrap_or(Value::Empty) } else { v };
}
}
}
/// [`call_lifted`] with every call's work added up against `budget` (see [`LIFTED_SCAN_BUDGET`]).
fn call_lifted_within(spec: &FnSpec, args: &[Arg], ctx: &mut dyn Ctx, budget: usize) -> Value {
let rejected = if spec.value_args.is_empty() { None } else { reject_multi_cell_refs(spec, args) };
let args = rejected.as_deref().unwrap_or(args);
let mut rows = 0usize;
@@ -150,42 +106,25 @@ fn call_lifted_within(spec: &FnSpec, args: &[Arg], ctx: &mut dyn Ctx, budget: us
return (spec.imp)(args, ctx);
}
let needles = rows.saturating_mul(cols);
// Every call is charged its repeated scan (at least 1), its own period loop, and the cells of
// an array result (built in full, though only its first cell is kept), so lifting a scalar
// argument can't multiply what one call may do into hours of work:
// `COUNTIF(big_range, SEQUENCE(1e4))`, `CUMIPMT(0.1, 1e7, 1, 1, SEQUENCE(1e4,,1e7,0), 0)`,
// `MUNIT(SEQUENCE(1e4,,3000,0))`.
let per_call = scan_len(spec, args).max(1);
let mut cell_args: Vec<Arg> = args.to_vec();
let mut spent = needles.saturating_mul(per_call);
if needles > MAX_CELLS || spent > budget {
if needles > MAX_CELLS || needles.saturating_mul(scan_len(spec, args)) > LIFTED_SCAN_BUDGET {
return Value::Error(CellError::Num);
}
// Period loops are known before any call runs, so a call over budget does none of them.
if PERIOD_LOOPS.contains(&spec.name) {
for r in 0..rows {
for c in 0..cols {
element_args(spec, args, r, c, &mut cell_args);
spent = spent.saturating_add(call_cost(spec, &cell_args));
if spent > budget {
return Value::Error(CellError::Num);
}
}
}
}
let mut cell_args: Vec<Arg> = args.to_vec();
let mut data = Vec::with_capacity(rows * cols);
for r in 0..rows {
for c in 0..cols {
element_args(spec, args, r, c, &mut cell_args);
for (i, a) in args.iter().enumerate() {
if spec.is_scalar_arg(i)
&& let Value::Array(arr) = &a.value
&& let Some(slot) = cell_args.get_mut(i)
{
let v = arr.get_broadcast(r, c);
slot.value = if let Value::Array(inner) = &v { inner.data.first().cloned().unwrap_or(Value::Empty) } else { v };
}
}
let out = (spec.imp)(&cell_args, ctx);
data.push(match out {
Value::Array(a) => {
spent = spent.saturating_add(a.data.len());
if spent > budget {
return Value::Error(CellError::Num);
}
a.data.first().cloned().unwrap_or(Value::Empty)
}
Value::Array(a) => a.data.first().cloned().unwrap_or(Value::Empty),
v => v,
});
}
@@ -270,52 +209,6 @@ mod tests {
assert_eq!(rows_of(&out), vec![vec![nv(-1.0)]; 1000]);
}
/// Security: lifting a criteria aggregate rescans its range for every criterion, so criteria
/// x range cells past the budget is `#NUM!` before any scan runs.
#[test]
fn lifted_criteria_aggregates_stop_past_their_work_budget() {
use gridcraft_core::Array;
let tall = |n: usize| av(Array::filled(n, 1, nv(-1.0)).into());
is_err(ev("COUNTIF", vec![tall(1_000_000), tall(501)]), CellError::Num);
is_err(ev("SUMIF", vec![tall(1_000_000), tall(501)]), CellError::Num);
is_err(ev("COUNTIFS", vec![tall(1_000_000), tall(501)]), CellError::Num);
// Two ranges of 250,000 cells each count together.
is_err(ev("SUMIFS", vec![tall(250_000), tall(250_000), tall(1001)]), CellError::Num);
let counts = ev("COUNTIF", vec![av(col(&[1.0, 2.0, 2.0])), av(col(&[2.0, 1.0, 5.0]))]);
assert_eq!(rows_of(&counts), vec![vec![nv(2.0)], vec![nv(1.0)], vec![nv(0.0)]]);
for name in super::CRITERIA.iter().chain(super::PERIOD_LOOPS) {
assert!(crate::lookup(name).is_some(), "{name} isn't registered");
}
}
/// Security: CUMIPMT and CUMPRINC loop over their periods once per lifted element; the
/// periods of all elements are added up first, so an over-budget call runs none of them.
#[test]
fn lifted_period_loops_stop_past_their_work_budget() {
use gridcraft_core::Array;
let ends = |len: usize, end: f64| av(Array::filled(len, 1, nv(end)).into());
let args = |len: usize, end: f64| vec![n(0.01), n(1e7), n(1000.0), n(1.0), ends(len, end), n(0.0)];
// 51 elements of 1e7 periods each is past 5e8.
is_err(ev("CUMIPMT", args(51, 1e7)), CellError::Num);
is_err(ev("CUMPRINC", args(51, 1e7)), CellError::Num);
let out = ev("CUMIPMT", args(2, 12.0));
assert_eq!(rows_of(&out).len(), 2);
assert!(matches!(rows_of(&out).first().and_then(|r| r.first()), Some(Value::Number(x)) if *x < 0.0));
}
/// Security: an array-producing function lifted over a computed array builds each element's
/// whole result before keeping one cell; their cells count against the budget.
#[test]
fn lifted_array_results_count_against_the_budget() {
use gridcraft_core::Array;
let spec = crate::lookup("MUNIT").unwrap_or_else(|| panic!("MUNIT isn't registered"));
let dims = |n: usize| [av(Array::filled(n, 1, nv(10.0)).into())];
// Ten 10x10 identity matrices are 1,000 cells.
let out = super::call_lifted_within(spec, &dims(10), &mut TestCtx::default(), 1_010);
assert_eq!(rows_of(&out), vec![vec![nv(1.0)]; 10]);
is_err(super::call_lifted_within(spec, &dims(11), &mut TestCtx::default(), 1_010), CellError::Num);
}
#[test]
fn value_params_reject_multi_cell_references() {
let s = FnSpec { value_args: &[true, false], ..spec() };