From 2ddcf6d2a1207cccef8836454a65a4cdfec1e9b2 Mon Sep 17 00:00:00 2001 From: NetterSilver <133535268+Silvertree2010@users.noreply.github.com> Date: Thu, 22 Oct 2026 20:08:56 +0200 Subject: [PATCH] Decode UTF-16 plain text files by BOM on import Found while testing crates/textimport/src/lib.rs:117 Co-Authored-By: Claude Haiku 5.5 Co-Authored-By: Claude Sonnet 5.5 --- crates/textimport/src/lib.rs | 12 +++++++++++- crates/textimport/src/tagged.rs | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/crates/textimport/src/lib.rs b/crates/textimport/src/lib.rs index ba9cb92e..56eac1a0 100644 --- a/crates/textimport/src/lib.rs +++ b/crates/textimport/src/lib.rs @@ -114,7 +114,7 @@ pub fn import(name: &str, bytes: &[u8]) -> Result { if l.ends_with(".doc") { return Err(ImportError::Unsupported("legacy Word .doc (save it as .docx)".into())); } - Ok(plain(&String::from_utf8_lossy(bytes))) + Ok(plain(&tagged::decode(bytes))) } /// Plain text: line breaks become paragraphs (CRLF / CR normalised). @@ -136,4 +136,14 @@ mod tests { assert!(is_text_file("Report.DOCX") && !is_text_file("a.png")); assert!(matches!(import("x.doc", b"\xD0\xCF"), Err(ImportError::Unsupported(_)))); } + + #[test] + fn plain_text_utf16() { + let mut le = vec![0xFF, 0xFE]; + le.extend("Hi there".encode_utf16().flat_map(u16::to_le_bytes)); + assert_eq!(import("a.txt", &le).unwrap().story.text, "Hi there"); + let mut be = vec![0xFE, 0xFF]; + be.extend("Hi there".encode_utf16().flat_map(u16::to_be_bytes)); + assert_eq!(import("a.txt", &be).unwrap().story.text, "Hi there"); + } } diff --git a/crates/textimport/src/tagged.rs b/crates/textimport/src/tagged.rs index 9bc84b76..18704cbc 100644 --- a/crates/textimport/src/tagged.rs +++ b/crates/textimport/src/tagged.rs @@ -72,7 +72,7 @@ pub fn tagged_text(doc: &Document, story: &Story) -> String { } /// Text of a Tagged Text file (ASCII, UTF-8 or UTF-16 by byte-order mark). -fn decode(bytes: &[u8]) -> String { +pub(crate) fn decode(bytes: &[u8]) -> String { if bytes.starts_with(&[0xFF, 0xFE]) || bytes.starts_with(&[0xFE, 0xFF]) { let le = bytes[0] == 0xFF; let units: Vec = bytes[2..]