From 7fece30d49ed166735b05302334c779fb155fa7b Mon Sep 17 00:00:00 2001
From: nihilist-narwhal <240798405+nihilist-narwhal@users.noreply.github.com>
Date: Sat, 10 Oct 2026 02:49:31 -0500
Subject: [PATCH] Remove Duplicates: compare only the part of the range with
 data

Given whole columns ("range":"D:D"), Remove Duplicates compared a
million blank rows and reported 1,048,573 removed duplicates; given the
whole sheet it ran for minutes, as it also did on a table resized to
the whole sheet. Rows and columns past the used range are blank, so
they are left out: the result is the same and the counts match the
data.
---
 crates/engine/src/cmd/data.rs | 3 +++
 crates/engine/src/tests.rs    | 4 ++++
 2 files changed, 7 insertions(+)

diff --git a/crates/engine/src/cmd/data.rs b/crates/engine/src/cmd/data.rs
index 7f8467e8a..38bece3bc 100644
--- a/crates/engine/src/cmd/data.rs
+++ b/crates/engine/src/cmd/data.rs
@@ -580,6 +580,9 @@ fn remove_duplicates(s: &mut Session, p: &Json) -> Result<Json> {
     let r = data_range(s, p)?;
     let d = s.doc()?;
     let sh = d.wb.active().ok_or(EngineError::NoDocument)?;
+    // Rows and columns past the data are blank: only the part with data is compared (whole
+    // columns compared a million blank rows and reported them as removed duplicates).
+    let r = sh.used_range().and_then(|u| r.intersection(&u)).unwrap_or(RangeRef::cell(r.start));
     let header = bool_param(p, "header").unwrap_or_else(|| guess_header(&d.wb, sh, r));
     let cols: Vec<u32> = p
         .get("columns")
diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs
index 9d0c7189f..e673485fb 100644
--- a/crates/engine/src/tests.rs
+++ b/crates/engine/src/tests.rs
@@ -181,6 +181,10 @@ fn whole_sheet_selection_stays_within_the_data() {
     s.execute("edit.fillUp", json!({})).unwrap();
     assert_eq!(v(&s, "A1"), Value::Number(5.0));
     assert!(s.execute("edit.fillDown", json!({"range": "A1:Z1000000"})).is_err(), "too large to fill");
+    s.execute("range.setValues", json!({"range": "D1", "values": [[1], [1], [2]]})).unwrap();
+    let r = s.execute("data.removeDuplicates", json!({"range": "D:D", "header": false})).unwrap();
+    assert_eq!((r["removed"].clone(), r["remaining"].clone()), (json!(1), json!(2)));
+    assert!(s.execute("data.removeDuplicates", json!({"range": "A1:XFD1048576", "header": false})).is_ok());
 }
 
 #[test]