fix(engine): preserve Unicode in variable data-set CSV (#1062)

crates/engine/src/variables_cmds.rs

@@ -310,29 +310,27 @@ fn split_csv(line: &str, delim: u8) -> Vec<String> {
     let mut out = Vec::new();
     let mut cur = String::new();
     let mut in_q = false;
-    let bytes = line.as_bytes();
-    let mut i = 0;
-    while i < bytes.len() {
-        let c = bytes[i];
+    let delim = char::from(delim);
+    let mut chars = line.chars().peekable();
+    while let Some(c) = chars.next() {
         if in_q {
-            if c == b'"' {
-                if i + 1 < bytes.len() && bytes[i + 1] == b'"' {
+            if c == '"' {
+                if chars.peek() == Some(&'"') {
                     cur.push('"');
-                    i += 1;
+                    let _ = chars.next();
                 } else {
                     in_q = false;
                 }
             } else {
-                cur.push(c as char);
+                cur.push(c);
             }
-        } else if c == b'"' {
+        } else if c == '"' {
             in_q = true;
         } else if c == delim {
             out.push(std::mem::take(&mut cur));
         } else {
-            cur.push(c as char);
+            cur.push(c);
         }
-        i += 1;
     }
     out.push(cur);
     out

crates/engine/src/variables_cmds/tests.rs

@@ -117,6 +117,100 @@ fn csv_import_and_apply() {
     assert_eq!(text_of(&s, title), "On Sale");
 }
 
+#[test]
+fn csv_fields_preserve_unicode_and_quoting() {
+    for (line, delimiter, expected) in [
+        ("Café,Привет,日本語,👩‍🎨,e\u{301}", b',', vec!["Café", "Привет", "日本語", "👩‍🎨", "e\u{301}"]),
+        ("\"Café, Привет\",\"日本語 \"\"quoted\"\" 👩‍🎨\",", b',', vec!["Café, Привет", "日本語 \"quoted\" 👩‍🎨", ""]),
+        ("Café;\"Привет;日本語\";👩‍🎨", b';', vec!["Café", "Привет;日本語", "👩‍🎨"]),
+        ("Café\t\"Привет\t日本語\"\t👩‍🎨", b'\t', vec!["Café", "Привет\t日本語", "👩‍🎨"]),
+        ("plain,\"with, comma\",\"a\"\"b\",,", b',', vec!["plain", "with, comma", "a\"b", "", ""]),
+        ("", b',', vec![""]),
+    ] {
+        assert_eq!(split_csv(line, delimiter), expected, "{line:?}");
+    }
+}
+
+#[test]
+fn csv_import_preserves_unicode_headers_names_and_text() {
+    let dir = tmp("unicode-text");
+    let csv = format!("{dir}/sets.csv");
+    for delimiter in [",", ";", "\t"] {
+        let (mut s, _photo, badge, title) = session();
+        s.execute(
+            "image.variables.define",
+            json!({"defs": [
+                {"name": "показать", "layer": badge.0, "type": "visibility"},
+                {"name": "заголовок café", "layer": title.0, "type": "textReplacement"},
+            ]}),
+        )
+        .unwrap();
+        let text = "Café, Привет; 日本語\t👩‍🎨 e\u{301} \"quoted\"";
+        std::fs::write(
+            &csv,
+            format!(
+                "DataSet{delimiter}показать{delimiter}заголовок café\r\nНабор 日本語{delimiter}true{delimiter}Café\r\n\"Строка \"\"été\"\"\"{delimiter}false{delimiter}\"{}\"\r\n",
+                text.replace('"', "\"\"")
+            ),
+        )
+        .unwrap();
+        let r = s.execute("file.import.variableDataSets", json!({"path": csv, "delimiter": delimiter})).unwrap();
+        assert_eq!(r["imported"], 2);
+        let rows = s.execute("variables.list", json!({})).unwrap();
+        assert_eq!(rows["dataSets"][0]["name"], "Набор 日本語");
+        assert_eq!(rows["dataSets"][0]["values"][1]["variable"], "заголовок café");
+        assert_eq!(rows["dataSets"][1]["values"][1]["value"], text);
+        s.execute("image.applyDataSet", json!({"name": "Набор 日本語"})).unwrap();
+        assert_eq!(text_of(&s, title), "Café");
+        s.execute("image.applyDataSet", json!({"name": "Строка \"été\""})).unwrap();
+        assert_eq!(text_of(&s, title), text);
+        assert!(!doc(&s).layer(badge).unwrap().visible);
+        assert!(s.undo());
+        assert_eq!(text_of(&s, title), "Café");
+        assert!(doc(&s).layer(badge).unwrap().visible);
+        assert!(s.redo());
+        assert_eq!(text_of(&s, title), text);
+    }
+    std::fs::remove_dir_all(dir).unwrap();
+}
+
+#[test]
+fn csv_import_preserves_unicode_pixel_paths() {
+    let (mut s, photo, _badge, _title) = session();
+    let dir = tmp("unicode-pixels");
+    let png = format!("{dir}/изображение,日本語.png");
+    let csv = format!("{dir}/sets.csv");
+    let mut src = Session::new();
+    src.execute("file.new", json!({"width": 8, "height": 8, "background": "#0000ff"})).unwrap();
+    let bytes = photocraft_io::export(&src.active().unwrap().doc, "png", &photocraft_io::ExportOptions::default()).unwrap().bytes;
+    std::fs::write(&png, bytes).unwrap();
+    s.execute("image.variables.define", json!({"defs": [{"name": "фото", "layer": photo.0, "type": "pixelReplacement", "method": "conform"}]})).unwrap();
+    std::fs::write(&csv, format!("DataSet,фото\nсиний,\"{png}\"\n")).unwrap();
+    s.execute("file.import.variableDataSets", json!({"path": csv})).unwrap();
+    let rows = s.execute("variables.list", json!({})).unwrap();
+    assert_eq!(rows["dataSets"][0]["values"][0]["value"], png);
+    s.execute("image.applyDataSet", json!({"name": "синий"})).unwrap();
+    let pixel = doc(&s).layer(photo).unwrap().surface().unwrap().pixel(32, 24);
+    assert_eq!(pixel, vec![0.0, 0.0, 1.0, 1.0]);
+    std::fs::remove_dir_all(dir).unwrap();
+}
+
+#[test]
+fn csv_import_errors_leave_existing_data_sets_unchanged() {
+    let (mut s, _photo, _badge, _title) = session();
+    s.execute("image.variables.dataSets", json!({"dataSets": [{"name": "Сохранённый набор", "values": []}]})).unwrap();
+    let dir = tmp("invalid-csv");
+    let csv = format!("{dir}/sets.csv");
+    for bytes in [b"".as_slice(), b"DataSet,headline\nrow,\xff".as_slice()] {
+        std::fs::write(&csv, bytes).unwrap();
+        let before = s.active().unwrap().doc.clone();
+        assert!(s.execute("file.import.variableDataSets", json!({"path": csv})).is_err());
+        assert!(Arc::ptr_eq(&before, &s.active().unwrap().doc));
+        assert_eq!(doc(&s).variables.data_sets[0].name, "Сохранённый набор");
+    }
+    std::fs::remove_dir_all(dir).unwrap();
+}
+
 #[test]
 fn export_data_sets_as_files() {
     let (mut s, _p, badge, title) = session();