Merge pull request #1609 from Silvertree2010/fix/h1332-placing-a-utf-16-plain-text-file

Decode UTF-16 plain text files by BOM on import

crates/textimport/src/lib.rs

@@ -114,7 +114,7 @@ pub fn import(name: &str, bytes: &[u8]) -> Result<Imported, ImportError> {
     if l.ends_with(".doc") {
         return Err(ImportError::Unsupported("legacy Word .doc (save it as .docx)".into()));
     }
-    Ok(plain(&String::from_utf8_lossy(bytes)))
+    Ok(plain(&tagged::decode(bytes)))
 }
 
 /// Plain text: line breaks become paragraphs (CRLF / CR normalised).
@@ -136,4 +136,14 @@ mod tests {
         assert!(is_text_file("Report.DOCX") && !is_text_file("a.png"));
         assert!(matches!(import("x.doc", b"\xD0\xCF"), Err(ImportError::Unsupported(_))));
     }
+
+    #[test]
+    fn plain_text_utf16() {
+        let mut le = vec![0xFF, 0xFE];
+        le.extend("Hi there".encode_utf16().flat_map(u16::to_le_bytes));
+        assert_eq!(import("a.txt", &le).unwrap().story.text, "Hi there");
+        let mut be = vec![0xFE, 0xFF];
+        be.extend("Hi there".encode_utf16().flat_map(u16::to_be_bytes));
+        assert_eq!(import("a.txt", &be).unwrap().story.text, "Hi there");
+    }
 }

crates/textimport/src/tagged.rs

@@ -72,7 +72,7 @@ pub fn tagged_text(doc: &Document, story: &Story) -> String {
 }
 
 /// Text of a Tagged Text file (ASCII, UTF-8 or UTF-16 by byte-order mark).
-fn decode(bytes: &[u8]) -> String {
+pub(crate) fn decode(bytes: &[u8]) -> String {
     if bytes.starts_with(&[0xFF, 0xFE]) || bytes.starts_with(&[0xFE, 0xFF]) {
         let le = bytes[0] == 0xFF;
         let units: Vec<u16> = bytes[2..]