Merge pull request #255 from nihilist-narwhal/fix-sort-and-duplicates

Sort and Remove Duplicates: compare text the way Excel does

crates/engine/src/cmd/data.rs

@@ -248,7 +248,7 @@ fn do_sort(s: &mut Session, r: RangeRef, header: bool, keys: Vec<SortKey>, by_co
                     let pos = |v: &Value| v.as_text().and_then(|t| list.iter().position(|x| x.eq_ignore_ascii_case(t))).unwrap_or(usize::MAX);
                     pos(va).cmp(&pos(vb))
                 } else if case && va.is_text() && vb.is_text() {
-                    va.as_text().unwrap_or("").cmp(vb.as_text().unwrap_or(""))
+                    sort_compare(va, vb).then_with(|| case_order(va.as_text().unwrap_or(""), vb.as_text().unwrap_or("")))
                 } else {
                     sort_compare(va, vb)
                 };
@@ -653,6 +653,12 @@ pub fn filter_values(wb: &Workbook, sheet: usize, col: u32) -> Vec<(String, bool
     seen.into_iter().map(|((_, t), on)| (t, on)).collect()
 }
 
+/// Case-sensitive tie-break for text that is equal ignoring case: as in Excel, lowercase sorts
+/// before uppercase at the first letter that differs in case.
+fn case_order(a: &str, b: &str) -> Ordering {
+    a.chars().zip(b.chars()).find(|(x, y)| x != y).map_or(Ordering::Equal, |(x, y)| y.is_lowercase().cmp(&x.is_lowercase()))
+}
+
 fn sort_key(v: &Value) -> (u8, i64, String) {
     match v {
         Value::Number(n) => (0, (n * 1e6) as i64, String::new()),
@@ -685,8 +691,11 @@ fn remove_duplicates(s: &mut Session, p: &Json) -> Result<Json> {
         let Some(sh) = cx.wb.sheet(sheet) else { return Ok(()) };
         let mut seen = std::collections::HashSet::new();
         let mut keep: Vec<u32> = Vec::new();
+        // Cells are compared by the text they show, as in Excel: the same date in two formats
+        // isn't a duplicate, while 1.01 and 1.02 shown as 1.0 are.
+        let shown = |c: CellRef| crate::display::format(&sh.value(c), cx.wb.styles.get(sh.style_id(c)).num_fmt.as_str(), &cx.wb).text;
         for row in start..=r.end.row {
-            let key: Vec<String> = cols.iter().map(|c| sh.value(CellRef::new(row, *c)).display().to_lowercase()).collect();
+            let key: Vec<String> = cols.iter().map(|c| shown(CellRef::new(row, *c)).to_lowercase()).collect();
             if seen.insert(key) {
                 keep.push(row);
             } else {

crates/engine/src/tests.rs

@@ -606,6 +606,27 @@ fn hidden_formulas_stay_hidden_on_protected_sheets() {
     assert_eq!(r["results"][0]["value"], "7");
 }
 
+#[test]
+fn case_sensitive_sort_puts_lowercase_first() {
+    let mut s = s();
+    s.execute("range.setValues", json!({"range": "A1", "values": [["b"], ["B"], ["Ab"], ["a"], ["aB"], ["A"]]})).unwrap();
+    s.execute("data.sort", json!({"range": "A1:A6", "header": false, "matchCase": true, "keys": [{"column": "A"}]})).unwrap();
+    let col: Vec<Value> = (1..=6).map(|r| v(&s, &format!("A{r}"))).collect();
+    assert_eq!(col, ["a", "A", "aB", "Ab", "b", "B"].map(Value::from));
+}
+
+#[test]
+fn remove_duplicates_compares_shown_text() {
+    let mut s = s();
+    s.execute("range.setValues", json!({"range": "A1", "values": [["Date", "Price"], ["2006-03-08", 1.01], ["2006-03-08", 1.02]]})).unwrap();
+    s.execute("home.numberFormat", json!({"range": "A3", "code": "mmm d, yyyy"})).unwrap();
+    s.execute("home.numberFormat", json!({"range": "B2:B3", "code": "0.0"})).unwrap();
+    let r = s.execute("data.removeDuplicates", json!({"range": "A1:A3", "header": true})).unwrap();
+    assert_eq!(r["removed"], 0);
+    let r = s.execute("data.removeDuplicates", json!({"range": "B1:B3", "header": true})).unwrap();
+    assert_eq!(r["removed"], 1);
+}
+
 #[test]
 fn formatting_and_styles() {
     let mut s = s();