Whole-sheet selection: Fill and Remove Duplicates stay within the data

With the whole sheet selected (Ctrl+A on an empty sheet), Fill Down,
Right, Up and Left tried to fill all 17 billion cells and ran out of
memory, and Remove Duplicates scanned every row for minutes. Fill Up
and Fill Left also took their source row or column before narrowing the
selection to the data, so they filled up to row 1,048,576 even when the
sheet had data.

Whole rows and columns now narrow to the used range before anything
else (nothing to do on an empty sheet), Remove Duplicates works on the
active cell of an empty sheet, and a fill of more than 5,000,000 cells
is refused, as Auto Fill already does.

crates/engine/src/cmd/data.rs

@@ -112,7 +112,8 @@ fn data_range(s: &Session, p: &Json) -> Result<RangeRef> {
     }
     let sel = d.selection.current();
     if !sel.is_single() {
-        let used = sh.used_range().unwrap_or(sel);
+        // An empty sheet has no data to work on: just the active cell.
+        let Some(used) = sh.used_range() else { return Ok(RangeRef::cell(d.selection.active)) };
         return Ok(sel.intersection(&used).unwrap_or(sel));
     }
     Ok(current_region(sh, d.selection.active))

crates/engine/src/cmd/edit.rs

@@ -1034,6 +1034,16 @@ fn clear(s: &mut Session, p: &Json, what: &str) -> Result<Json> {
 fn fill_dir(s: &mut Session, p: &Json, dr: i64, dc: i64) -> Result<Json> {
     let sheet = target_sheet(s, p)?;
     let mut r = target_range(s, p)?;
+    // Whole rows or columns fill only where the sheet has content (nothing on an empty sheet).
+    if r.is_full_cols() || r.is_full_rows() {
+        match s.doc()?.wb.sheet(sheet).and_then(|sh| sh.used_range()).and_then(|u| r.intersection(&u)) {
+            Some(u) => r = u,
+            None => return Ok(Json::Null),
+        }
+    }
+    if r.count() > 5_000_000 {
+        return Err(bad("edit.fill", "range too large"));
+    }
     // With one row selected, Fill Down copies from the row above (Excel behaviour).
     if dr == 1 && r.height() == 1 {
         r.start.row = r.start.row.saturating_sub(1);
@@ -1050,8 +1060,6 @@ fn fill_dir(s: &mut Session, p: &Json, dr: i64, dc: i64) -> Result<Json> {
     } else {
         RangeRef::new(CellRef::new(r.start.row, r.end.col), r.end)
     };
-    let used = s.doc()?.wb.sheet(sheet).and_then(|sh| sh.used_range());
-    let r = if r.is_full_cols() || r.is_full_rows() { used.and_then(|u| r.intersection(&u)).unwrap_or(r) } else { r };
     edit(s, |cx| {
         crate::fill::fill(cx, sheet, src, r, crate::fill::FillMode::Copy)?;
         Ok(Json::Null)

crates/engine/src/tests.rs

@@ -167,6 +167,22 @@ fn every_command_survives_empty_params() {
     }
 }
 
+#[test]
+fn whole_sheet_selection_stays_within_the_data() {
+    // On an empty sheet with everything selected these filled or scanned every cell of the
+    // sheet, running out of memory or time.
+    let mut s = s();
+    s.execute("edit.selectAll", json!({})).unwrap();
+    for cmd in ["edit.fillDown", "edit.fillRight", "edit.fillUp", "edit.fillLeft", "data.removeDuplicates"] {
+        s.execute(cmd, json!({})).unwrap();
+    }
+    s.execute("range.setValues", json!({"range": "A1", "values": [[1, 2], [3, 4], [5, 6]]})).unwrap();
+    s.execute("selection.set", json!({"range": "A1:XFD1048576"})).unwrap();
+    s.execute("edit.fillUp", json!({})).unwrap();
+    assert_eq!(v(&s, "A1"), Value::Number(5.0));
+    assert!(s.execute("edit.fillDown", json!({"range": "A1:Z1000000"})).is_err(), "too large to fill");
+}
+
 #[test]
 fn parity_counts() {
     let (done, total) = crate::catalog::parity();