Functions: database functions evaluate computed criteria (part of #737)

A formula under a criteria label that names no field is evaluated for each record: its relative references move from the first record to that record, and the database labels name the record's fields, as in Excel. The calc evaluator passes the criteria reference to the database functions through Ctx::computed_criterion.

crates/calc/src/eval.rs

@@ -88,6 +88,8 @@ pub struct Evaluator<'h> {
 
 struct FnCtx<'a, 'h> {
     ev: &'a mut Evaluator<'h>,
+    /// The criteria argument of a database function, when it is a reference.
+    criteria: Option<Area>,
 }
 
 impl Ctx for FnCtx<'_, '_> {
@@ -100,6 +102,24 @@ impl Ctx for FnCtx<'_, '_> {
     fn random(&mut self) -> f64 {
         self.ev.host.random()
     }
+    /// Excel's "formula as criteria": the formula is written for the first record, so its relative
+    /// references move down to `record`'s row, and the database's labels name its fields.
+    fn computed_criterion(&mut self, row: usize, col: usize, db: &Array, record: usize) -> Option<bool> {
+        let area = self.criteria?;
+        let at = area.range.start.offset(row as i64, col as i64).filter(|c| area.range.contains(*c))?;
+        let expr = self.ev.host.workbook().sheet(area.sheet)?.cell(at)?.formula.as_ref()?.expr_arc()?;
+        let shifted = gridcraft_formula::adjust::shift_relative((*expr).clone(), record as i64 - 1, 0);
+        let env = (0..db.cols)
+            .filter_map(|c| match db.get(0, c) {
+                Some(Value::Text(t)) if !t.trim().is_empty() => {
+                    Some((t.trim().to_string(), Ev::V(db.get(record, c).cloned().unwrap_or(Value::Empty))))
+                }
+                _ => None,
+            })
+            .collect();
+        let mut ev = Evaluator { host: &mut *self.ev.host, sheet: area.sheet, at, env, depth: self.ev.depth };
+        Some(matches!(ev.value(&shifted).scalar().to_bool(), Ok(true)))
+    }
 }
 
 impl<'h> Evaluator<'h> {
@@ -579,7 +599,7 @@ impl<'h> Evaluator<'h> {
                         let value = self.value(a);
                         evaluated.push(Arg { value, from_ref: false });
                     }
-                    let mut ctx = FnCtx { ev: self };
+                    let mut ctx = FnCtx { ev: self, criteria: None };
                     return gridcraft_functions::call(spec, &evaluated, &mut ctx).pipe(Ev::V);
                 }
                 let i = match index.to_number() {
@@ -762,8 +782,17 @@ impl<'h> Evaluator<'h> {
         let mut evaluated = Vec::with_capacity(args.len());
         // SUMIF/AVERAGEIF: the size of the criteria range (argument 1), which the sum range takes.
         let mut criteria_shape = None;
+        // Database functions: the criteria range (argument 2), whose formulas are computed criteria.
+        let mut criteria = None;
         for (i, a) in args.iter().enumerate() {
             let mut ev = self.eval(a);
+            if spec.category == gridcraft_functions::Category::Database
+                && i == 2
+                && let Ev::R(areas) = &ev
+                && let [area] = areas.as_slice()
+            {
+                criteria = Some(*area);
+            }
             if resizes_sum_range(name, args.len()) {
                 if i == 0 {
                     criteria_shape = ev_shape(&ev);
@@ -791,7 +820,7 @@ impl<'h> Evaluator<'h> {
             };
             evaluated.push(Arg { value, from_ref });
         }
-        let mut ctx = FnCtx { ev: self };
+        let mut ctx = FnCtx { ev: self, criteria };
         gridcraft_functions::call(spec, &evaluated, &mut ctx).pipe(Ev::V)
     }
 
@@ -845,13 +874,13 @@ impl<'h> Evaluator<'h> {
         };
         let args = [Arg { value, from_ref }];
         if kind != Agg::Count {
-            match gridcraft_functions::call(sum, &args, &mut FnCtx { ev: self }) {
+            match gridcraft_functions::call(sum, &args, &mut FnCtx { ev: self, criteria: None }) {
                 Value::Number(n) => acc.sum += n,
                 other => return Err(Ev::V(other)),
             }
         }
         if kind != Agg::Sum {
-            match gridcraft_functions::call(count, &args, &mut FnCtx { ev: self }) {
+            match gridcraft_functions::call(count, &args, &mut FnCtx { ev: self, criteria: None }) {
                 Value::Number(n) => acc.count = acc.count.saturating_add(n as u64),
                 other => return Err(Ev::V(other)),
             }
@@ -1221,7 +1250,7 @@ impl<'h> Evaluator<'h> {
                         }
                     }
                 }
-                let mut ctx = FnCtx { ev: self };
+                let mut ctx = FnCtx { ev: self, criteria: None };
                 return match gridcraft_functions::call(spec, &xargs, &mut ctx) {
                     Value::Number(p) if p >= 1.0 => {
                         let k = p as u32 - 1;

crates/calc/src/tests.rs

@@ -968,3 +968,55 @@ fn trim_references() {
     t.set("K20", "=SUM(Q1:.Q9)");
     assert_eq!(t.get("K20"), Value::Error(CellError::Calc));
 }
+
+/// Expected results are Excel's, from Apache POI's `DGet.xls` (rows 46-49): a criteria cell holding
+/// a formula under a label that names no field is evaluated for every record.
+#[test]
+fn database_functions_with_computed_criteria() {
+    let mut t = T::new();
+    let db = [
+        ["name", "age", "size", "headgear", "glasses"],
+        ["Huey", "6", "45", "cap~", "FALSE"],
+        ["Dewey", "7", "fourty-six", "cap?", "FALSE"],
+        ["Louie", "8", "47", "cap*", "FALSE"],
+        ["Donald", "32", "70", "sailor's hat", "FALSE"],
+        ["Scrooge", "62", "65", "top hat", "TRUE"],
+    ];
+    for (r, row) in db.iter().enumerate() {
+        for (c, v) in row.iter().enumerate() {
+            t.set(&format!("{}{}", ["F", "G", "H", "I", "J"][c], r + 2), v);
+        }
+    }
+    t.set("G45", "7");
+    let cases = [
+        // Relative references are to the first record's row and move to each record's row;
+        // `$G$45` stays put.
+        ("G", "=G2<$G$45", "Dewey"),
+        // Column labels stand for the record's fields.
+        ("H", "=glasses=TRUE", "Scrooge"),
+        ("I", "=age<7", "Huey"),
+        ("J", "=size/7=10", "Donald"),
+        // An absolute reference to a cell that is not the first record's.
+        ("K", "=H3=$L$1", "Donald"),
+    ];
+    t.set("L1", "70");
+    for (i, (col, formula, _)) in cases.iter().enumerate() {
+        t.set(&format!("{col}46"), "Formula");
+        t.set(&format!("{col}47"), formula);
+        t.set(&format!("B{}", 46 + i), &format!("=DGET($F$2:$J$7,\"name\",{col}46:{col}47)"));
+    }
+    for (i, (_, formula, want)) in cases.iter().enumerate() {
+        assert_eq!(t.get(&format!("B{}", 46 + i)), Value::text(*want), "{formula}");
+    }
+    // Every database function takes them, ANDed with the other criteria of their row.
+    // Text sorts after numbers, so Dewey's "fourty-six" is > 40.
+    t.set("L47", "=size>40");
+    t.set("M46", "age");
+    t.set("M47", ">6");
+    t.set("B51", "=DCOUNT($F$2:$J$7,\"age\",I46:I47)");
+    t.set("B52", "=DSUM($F$2:$J$7,\"age\",G46:G47)");
+    t.set("B53", "=DCOUNTA($F$2:$J$7,\"name\",L46:M47)");
+    assert_eq!(t.num("B51"), 1.0);
+    assert_eq!(t.num("B52"), 7.0);
+    assert_eq!(t.num("B53"), 4.0);
+}

crates/functions/src/database.rs

@@ -4,7 +4,9 @@
 //! The database and the criteria are arrays whose first row holds column labels. Each criteria
 //! row is a set of conditions that must all hold (AND); a record matches when any criteria row
 //! holds (OR). An empty criteria cell matches anything. Plain text conditions match values that
-//! begin with the text, as in Excel's advanced filter, so an empty string matches any text.
+//! begin with the text, as in Excel's advanced filter, so an empty string matches any text. A
+//! formula under a label that names no field is a computed criterion, which the host evaluates
+//! for each record ([`Ctx::computed_criterion`]).
 
 use std::borrow::Cow;
 
@@ -64,9 +66,10 @@ fn cell_criterion(v: &Value) -> Option<Criterion> {
 }
 
 /// Rows (1-based within the database, excluding the header) of matching records.
-fn matching_records(db: &Array, crit: &Array) -> Vec<usize> {
-    // Per criteria row: list of (database column or None when the label is unknown, criterion).
-    let mut rows: Vec<Vec<(Option<usize>, Criterion)>> = Vec::new();
+fn matching_records(db: &Array, crit: &Array, ctx: &mut dyn Ctx) -> Vec<usize> {
+    // Per criteria row: its index and a list of (criteria column, database column or None when
+    // the label is unknown, criterion).
+    let mut rows: Vec<(usize, Vec<(usize, Option<usize>, Criterion)>)> = Vec::new();
     for r in 1..crit.rows {
         let mut conds = Vec::new();
         for c in 0..crit.cols {
@@ -77,18 +80,20 @@ fn matching_records(db: &Array, crit: &Array) -> Vec<usize> {
                 Some(Value::Number(n)) => find_column(db, &gridcraft_core::number_to_text(*n)),
                 _ => None,
             };
-            conds.push((col, cr));
+            conds.push((c, col, cr));
         }
-        rows.push(conds);
+        rows.push((r, conds));
     }
     let all_match = rows.is_empty();
     (1..db.rows)
-        .filter(|&r| {
+        .filter(|&rec| {
             all_match
-                || rows.iter().any(|conds| {
-                    conds.iter().all(|(col, cr)| match col {
-                        Some(c) => cr.matches(db.get(r, *c).unwrap_or(&Value::Empty)),
-                        None => false,
+                || rows.iter().any(|(r, conds)| {
+                    conds.iter().all(|(c, col, cr)| match col {
+                        Some(f) => cr.matches(db.get(rec, *f).unwrap_or(&Value::Empty)),
+                        // Under a label that names no field, only a formula (a computed
+                        // criterion) can hold.
+                        None => ctx.computed_criterion(*r, *c, db, rec).unwrap_or(false),
                     })
                 })
         })
@@ -101,7 +106,7 @@ struct Selection<'a> {
     records: Vec<usize>,
 }
 
-fn select(a: &[Arg]) -> R<Selection<'_>> {
+fn select<'a>(a: &'a [Arg], ctx: &mut dyn Ctx) -> R<Selection<'a>> {
     let dbv = &a.first().ok_or(CellError::Value)?.value;
     let critv = &a.get(2).ok_or(CellError::Value)?.value;
     if let Value::Error(e) = dbv {
@@ -113,31 +118,31 @@ fn select(a: &[Arg]) -> R<Selection<'_>> {
     let db = as_array(dbv);
     let crit = as_array(critv);
     let field = field_index(&db, a)?;
-    let records = matching_records(&db, &crit);
+    let records = matching_records(&db, &crit, ctx);
     Ok(Selection { db, field, records })
 }
 
 /// Numbers of the field among matching records (field required).
-fn field_numbers(a: &[Arg]) -> R<Vec<f64>> {
-    let s = select(a)?;
+fn field_numbers(a: &[Arg], c: &mut dyn Ctx) -> R<Vec<f64>> {
+    let s = select(a, c)?;
     let f = s.field.ok_or(CellError::Value)?;
     Ok(s.records.iter().filter_map(|&r| s.db.get(r, f).and_then(Value::as_f64)).collect())
 }
 
-fn dsum(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    num_val(field_numbers(a)?.iter().sum())
+fn dsum(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    num_val(field_numbers(a, c)?.iter().sum())
 }
 
-fn daverage(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    let v = field_numbers(a)?;
+fn daverage(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    let v = field_numbers(a, c)?;
     if v.is_empty() {
         return Err(CellError::Div0);
     }
     num_val(v.iter().sum::<f64>() / v.len() as f64)
 }
 
-fn dcount(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    let s = select(a)?;
+fn dcount(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    let s = select(a, c)?;
     let n = match s.field {
         Some(f) => s.records.iter().filter(|&&r| matches!(s.db.get(r, f), Some(Value::Number(_)))).count(),
         None => s.records.len(),
@@ -145,8 +150,8 @@ fn dcount(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
     num_val(n as f64)
 }
 
-fn dcounta(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    let s = select(a)?;
+fn dcounta(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    let s = select(a, c)?;
     let n = match s.field {
         Some(f) => s.records.iter().filter(|&&r| !matches!(s.db.get(r, f), None | Some(Value::Empty))).count(),
         None => s.records.len(),
@@ -154,8 +159,8 @@ fn dcounta(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
     num_val(n as f64)
 }
 
-fn dget(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    let s = select(a)?;
+fn dget(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    let s = select(a, c)?;
     let f = s.field.ok_or(CellError::Value)?;
     // Records whose field is blank (or "") don't count, as in Excel.
     let mut found =
@@ -167,18 +172,18 @@ fn dget(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
     }
 }
 
-fn dmax(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    let v = field_numbers(a)?;
+fn dmax(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    let v = field_numbers(a, c)?;
     num_val(v.iter().copied().reduce(f64::max).unwrap_or(0.0))
 }
 
-fn dmin(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    let v = field_numbers(a)?;
+fn dmin(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    let v = field_numbers(a, c)?;
     num_val(v.iter().copied().reduce(f64::min).unwrap_or(0.0))
 }
 
-fn dproduct(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    let v = field_numbers(a)?;
+fn dproduct(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    let v = field_numbers(a, c)?;
     if v.is_empty() {
         return num_val(0.0);
     }
@@ -196,20 +201,20 @@ fn variance(v: &[f64], sample: bool) -> R<f64> {
     Ok(ss / if sample { n - 1.0 } else { n })
 }
 
-fn dvar(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    num_val(variance(&field_numbers(a)?, true)?)
+fn dvar(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    num_val(variance(&field_numbers(a, c)?, true)?)
 }
 
-fn dvarp(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    num_val(variance(&field_numbers(a)?, false)?)
+fn dvarp(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    num_val(variance(&field_numbers(a, c)?, false)?)
 }
 
-fn dstdev(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    num_val(variance(&field_numbers(a)?, true)?.sqrt())
+fn dstdev(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    num_val(variance(&field_numbers(a, c)?, true)?.sqrt())
 }
 
-fn dstdevp(a: &[Arg], _c: &mut dyn Ctx) -> R<Value> {
-    num_val(variance(&field_numbers(a)?, false)?.sqrt())
+fn dstdevp(a: &[Arg], c: &mut dyn Ctx) -> R<Value> {
+    num_val(variance(&field_numbers(a, c)?, false)?.sqrt())
 }
 
 pub(crate) fn specs() -> Vec<FnSpec> {

crates/functions/src/lib.rs

@@ -15,7 +15,7 @@
 use std::collections::HashMap;
 use std::sync::OnceLock;
 
-use gridcraft_core::{CellError, DateSystem, Value};
+use gridcraft_core::{Array, CellError, DateSystem, Value};
 
 /// Builds a [`FnSpec`] whose implementation returns `Result<Value, CellError>`.
 ///
@@ -105,6 +105,12 @@ pub trait Ctx {
     fn now_serial(&self) -> f64;
     /// Uniform in [0, 1).
     fn random(&mut self) -> f64;
+    /// For the database functions: whether the formula in the criteria cell at (`row`, `col`) holds
+    /// for `record` (a row of `db` below its labels). `None` when that cell holds no formula or the
+    /// criteria didn't come from a reference.
+    fn computed_criterion(&mut self, _row: usize, _col: usize, _db: &Array, _record: usize) -> Option<bool> {
+        None
+    }
 }
 
 pub type FnImpl = fn(&[Arg], &mut dyn Ctx) -> Value;