fix(crypt): R2–R4 passwords with non-ASCII characters open (PDFDocEncoding) (#905)

Revisions 2–4 of the standard security handler take the password in
PDFDocEncoding (ISO 32000-2 §7.6.4.3.2, Algorithm 2). PdfCraft authenticates
an encrypted file twice, and neither did that:
- pdfcraft-crypt took Latin-1 and dropped every other character, so "ı"
  (0x9A) and "€" (0xA0) vanished and "ılık€" became "lk";
- the renderer (vendored hayro-syntax, "TODO: Convert to PDFDocEncoding")
  used the UTF-8 bytes, so any non-ASCII password failed, even "Çok güzel",
  which Latin-1 and PDFDocEncoding write alike;
- the engine handed the renderer an owner-recovered user password as Latin-1
  characters, which the renderer then read as UTF-8 again.
Files that qpdf or Acrobat protected with such passwords could not be
opened with the right password.

- pdfcraft-crypt encodes R2–R4 passwords in PDFDocEncoding and, for a
  password with characters PDFDocEncoding lacks (ğ, ş, İ), also tries its
  UTF-8 bytes, which qpdf writes for those. The PDFDocEncoding table moves
  here from pdfcraft-cos (which depends on this crate) and cos uses it.
- hayro-syntax (vendored patch 14) tries an R2–R4 password as given, then
  in PDFDocEncoding.
- The engine passes a recovered user password to the renderer as UTF-8 when
  its bytes are UTF-8 and as PDFDocEncoding otherwise, so the renderer gets
  exactly those bytes back.

Co-authored-by: Brandon Thomas <echelon@gmail.com>

crates/cos/src/lib.rs

@@ -25,7 +25,7 @@ pub use bytes::Bytes;
 pub use document::{Document, Revision, SourceIdentity, XrefEntry};
 pub use object::{Dict, MAX_DECODED, Name, ObjRef, Object, PdfString, Stream};
 pub use parser::{Lexer, parse_indirect};
-pub use pdfcraft_crypt::{Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler};
+pub use pdfcraft_crypt::{Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler, pdfdoc_byte, pdfdoc_char};
 pub use writer::{SaveOptions, pdf_date, serialize, write_full, write_incremental};
 
 #[derive(Debug, thiserror::Error, Clone, PartialEq)]

crates/cos/src/object.rs

@@ -66,27 +66,13 @@ impl PdfString {
             b => match std::str::from_utf8(b) {
                 // Pure ASCII is identical either way; only non-ASCII UTF-8 needs the switch.
                 Ok(utf8) if !b.is_ascii() => utf8.to_owned(),
-                _ => b.iter().map(|&c| pdfdoc_char(c)).collect(),
+                _ => b.iter().map(|&c| pdfcraft_crypt::pdfdoc_char(c)).collect(),
             },
         };
         if s.starts_with('\u{FEFF}') { s.trim_start_matches('\u{FEFF}').to_owned() } else { s }
     }
 }
 
-/// PDFDocEncoding → Unicode (ISO 32000-2 Annex D). Bytes 0x18–0x1F and 0x80–0xA0 differ from Latin-1.
-fn pdfdoc_char(c: u8) -> char {
-    const HIGH: [char; 33] = [
-        '•', '†', '‡', '…', '—', '–', 'ƒ', '⁄', '‹', '›', '−', '‰', '„', '“', '”', '‘', '’', '‚', '™', 'fi', 'fl', 'Ł', 'Œ', 'Š', 'Ÿ', 'Ž', 'ı', 'ł',
-        'œ', 'š', 'ž', '\u{FFFD}', '€',
-    ];
-    const LOW: [char; 8] = ['˘', 'ˇ', 'ˆ', '˙', '˝', '˛', '˚', '˜'];
-    match c {
-        0x18..=0x1F => LOW[(c - 0x18) as usize],
-        0x80..=0xA0 => HIGH[(c - 0x80) as usize],
-        _ => c as char,
-    }
-}
-
 /// A name object (without the leading `/`, with `#xx` escapes already decoded).
 pub type Name = Vec<u8>;
 

crates/crypt/src/lib.rs

@@ -151,15 +151,19 @@ impl SecurityHandler {
         let mut recovered_user = None;
         let (key, auth) = match dict.r {
             2..=4 => {
-                let legacy = legacy_password(pw);
-                if let Some((k, user)) = owner_key_r4(&dict, id0, &legacy) {
-                    recovered_user = Some(user);
-                    (k, Auth::Owner)
-                } else if let Some(k) = user_key_r4(&dict, id0, &legacy) {
-                    (k, Auth::User)
-                } else {
-                    return Err(CryptError::WrongPassword);
+                let mut found = None;
+                for legacy in legacy_candidates(pw) {
+                    if let Some((k, user)) = owner_key_r4(&dict, id0, &legacy) {
+                        recovered_user = Some(user);
+                        found = Some((k, Auth::Owner));
+                    } else if let Some(k) = user_key_r4(&dict, id0, &legacy) {
+                        found = Some((k, Auth::User));
+                    }
+                    if found.is_some() {
+                        break;
+                    }
                 }
+                found.ok_or(CryptError::WrongPassword)?
             }
             5 | 6 => {
                 let pw = sasl_password(pw);
@@ -300,9 +304,48 @@ impl SecurityHandler {
 
 // ── Passwords ──────────────────────────────────────────────────────────────────────────────────
 
-/// Revisions 2–4 use PDFDocEncoding passwords; characters outside Latin-1 are dropped.
+/// PDFDocEncoding (ISO 32000-2 Annex D) where it differs from Latin-1: bytes 0x18–0x1F and
+/// 0x80–0xA0 (0x9F is undefined). Here rather than in pdfcraft-cos, which decodes text strings
+/// with it, because R2–R4 passwords need it too and cos depends on this crate.
+const PDFDOC_LOW: [char; 8] = ['˘', 'ˇ', 'ˆ', '˙', '˝', '˛', '˚', '˜'];
+const PDFDOC_HIGH: [char; 33] = [
+    '•', '†', '‡', '…', '—', '–', 'ƒ', '⁄', '‹', '›', '−', '‰', '„', '“', '”', '‘', '’', '‚', '™', 'fi', 'fl', 'Ł', 'Œ', 'Š', 'Ÿ', 'Ž', 'ı', 'ł', 'œ',
+    'š', 'ž', '\u{FFFD}', '€',
+];
+
+/// PDFDocEncoding → Unicode.
+pub fn pdfdoc_char(c: u8) -> char {
+    match c {
+        0x18..=0x1F => PDFDOC_LOW[usize::from(c - 0x18)],
+        0x80..=0xA0 => PDFDOC_HIGH[usize::from(c - 0x80)],
+        _ => char::from(c),
+    }
+}
+
+/// Unicode → PDFDocEncoding, `None` for a character it lacks (ğ, ş, İ, Cyrillic, CJK, …).
+pub fn pdfdoc_byte(c: char) -> Option<u8> {
+    let at = |table: &[char], base: u8| table.iter().position(|&t| t == c && t != '\u{FFFD}').and_then(|i| u8::try_from(i).ok()).map(|i| base + i);
+    at(&PDFDOC_LOW, 0x18).or_else(|| at(&PDFDOC_HIGH, 0x80)).or_else(|| match u32::from(c) {
+        0x18..=0x1F | 0x80..=0xA0 => None,
+        n => u8::try_from(n).ok(),
+    })
+}
+
+/// Revisions 2–4: the password in PDFDocEncoding (§7.6.4.3.2, Algorithm 2), at most 32 bytes.
+/// Characters PDFDocEncoding lacks are dropped.
 fn legacy_password(pw: &str) -> Vec<u8> {
-    pw.chars().filter_map(|c| u8::try_from(u32::from(c)).ok()).take(32).collect()
+    pw.chars().filter_map(pdfdoc_byte).take(32).collect()
+}
+
+/// The bytes an R2–R4 password may have been written as: PDFDocEncoding, as the specification
+/// asks; and for a password with characters PDFDocEncoding lacks (ğ, ş, İ, …), which it gives
+/// no bytes, also its UTF-8 bytes, which qpdf and other tools write for such passwords.
+fn legacy_candidates(pw: &str) -> Vec<Vec<u8>> {
+    let mut out = vec![legacy_password(pw)];
+    if pw.chars().any(|c| pdfdoc_byte(c).is_none()) {
+        out.push(pw.bytes().take(32).collect());
+    }
+    out
 }
 
 /// Revisions 5–6: SASLprep (RFC 4013), UTF-8, at most 127 bytes.

crates/crypt/src/tests.rs

@@ -163,3 +163,75 @@ fn owner_authentication_recovers_the_user_password() {
     let d = new(Algorithm::Rc4_128, "", "owner", -1).dict().clone();
     assert_eq!(SecurityHandler::open(d, ID, Some("owner")).unwrap().recovered_user_password().as_deref(), Some(&b""[..]));
 }
+
+/// /O, /U and the first /ID of files qpdf 12.4.2 wrote (`--static-id`), with the file keys
+/// `qpdf --show-encryption-key` reports.
+fn qpdf_file(r: i64, o: &str, u: &str) -> EncryptDict {
+    let mut d = EncryptDict {
+        filter: b"Standard".to_vec(),
+        v: if r == 4 { 4 } else { 2 },
+        r,
+        length_bits: 128,
+        o: hex(o),
+        u: hex(u),
+        p: -4,
+        encrypt_metadata: true,
+        ..Default::default()
+    };
+    if r == 4 {
+        d.crypt_filters = vec![(b"StdCF".to_vec(), Method::Aes128)];
+        d.stm_f = b"StdCF".to_vec();
+        d.str_f = b"StdCF".to_vec();
+    }
+    d
+}
+
+const QPDF_ID: &str = "56b57383aafa8314f3f98222995f5604";
+
+#[test]
+fn legacy_passwords_are_pdfdoc_encoded_with_a_utf8_fallback() {
+    let id = hex(QPDF_ID);
+    // R3 RC4: user "ılık€" in PDFDocEncoding (ı = 0x9A, € = 0xA0); owner "Öğretmen", which
+    // PDFDocEncoding can't hold, so qpdf wrote its UTF-8 bytes.
+    let r3 = qpdf_file(
+        3,
+        "d519aa8120ce8c54812b45e5391916fed167b1c84961d90c4ee4c182ad83534b",
+        "4985c03ddfa3eeb8a90c343512913ec40021446990b9e4114071a4d9104984c1",
+    );
+    let user = SecurityHandler::open(r3.clone(), &id, Some("ılık€")).unwrap();
+    assert_eq!((user.auth(), user.file_key().to_vec()), (Auth::User, hex("3564ffcea186122595ad55ee157da71f")));
+    let owner = SecurityHandler::open(r3.clone(), &id, Some("Öğretmen")).unwrap();
+    assert_eq!((owner.auth(), owner.file_key().to_vec()), (Auth::Owner, hex("3564ffcea186122595ad55ee157da71f")));
+    assert_eq!(owner.recovered_user_password(), Some(vec![0x9A, b'l', 0x9A, b'k', 0xA0]));
+    // Dropping ı and € (the old Latin-1 conversion) or a near miss doesn't open it.
+    for wrong in ["lk", "ilik€", "ılık", "Ogretmen"] {
+        assert_eq!(SecurityHandler::open(r3.clone(), &id, Some(wrong)).unwrap_err(), CryptError::WrongPassword, "{wrong}");
+    }
+    // R4 AES-128: user "şifre" (UTF-8 bytes), owner "owner".
+    let r4 = qpdf_file(
+        4,
+        "bb4f8f4bd2236569e765906caf64e4429a4c20d6e996fdef963e9b5080f9e083",
+        "4a965fb91882772cd09ce8bbff2c33270021446990b9e4114071a4d9104984c1",
+    );
+    let user = SecurityHandler::open(r4.clone(), &id, Some("şifre")).unwrap();
+    assert_eq!((user.auth(), user.file_key().to_vec()), (Auth::User, hex("9f19d465cfd8e525d32aa0bdf10acd06")));
+    assert_eq!(SecurityHandler::open(r4.clone(), &id, Some("owner")).unwrap().auth(), Auth::Owner);
+    assert_eq!(SecurityHandler::open(r4, &id, Some("ifre")).unwrap_err(), CryptError::WrongPassword);
+}
+
+#[test]
+fn pdfdoc_encoding_round_trips() {
+    for b in 0..=255u8 {
+        if b != 0x9F {
+            assert_eq!(pdfdoc_byte(pdfdoc_char(b)), Some(b), "{b:#04x}");
+        }
+    }
+    assert_eq!(pdfdoc_byte('ı'), Some(0x9A));
+    assert_eq!(pdfdoc_byte('€'), Some(0xA0));
+    assert_eq!(pdfdoc_byte('ç'), Some(0xE7));
+    for c in ['ğ', 'ş', 'İ', '\u{A0}', '\u{85}', 'Ж', '日'] {
+        assert_eq!(pdfdoc_byte(c), None, "{c}");
+    }
+    assert_eq!(legacy_candidates("Çok"), [vec![0xC7, b'o', b'k']]);
+    assert_eq!(legacy_candidates("şifre"), [b"ifre".to_vec(), "şifre".as_bytes().to_vec()]);
+}

crates/engine/src/lib.rs

@@ -1812,6 +1812,17 @@ fn keys_after(edit: &Edit) -> Option<Keys> {
     }
 }
 
+/// The recovered user password of an R2–R4 file as text for the renderer, which tries a password
+/// as its UTF-8 bytes and then in PDFDocEncoding: UTF-8 bytes (as some writers store passwords
+/// PDFDocEncoding can't hold, such as "şifre") stay UTF-8, and other bytes are read as
+/// PDFDocEncoding, so either way the renderer gets back exactly these bytes.
+fn renderer_password(bytes: &[u8]) -> String {
+    match std::str::from_utf8(bytes) {
+        Ok(text) => text.to_owned(),
+        Err(_) => bytes.iter().map(|b| pdfcraft_cos::pdfdoc_char(*b)).collect(),
+    }
+}
+
 /// The last-resort guard (AGENTS.md §4): run `f`, turning a panic that escapes it into an error
 /// message, so one bad file or edit can't take the app and its other documents down. It is a
 /// safety net for bugs, not a substitute for returning errors.
@@ -2129,7 +2140,7 @@ impl Session {
                     Ok(Ok(d)) => d.security().and_then(|s| s.recovered_user_password()),
                     _ => None,
                 };
-                let user: String = user.ok_or(OpenError::WrongPassword)?.iter().map(|b| char::from(*b)).collect();
+                let user = renderer_password(&user.ok_or(OpenError::WrongPassword)?);
                 (inspect(bytes.clone(), Some(&user))?, Some(user))
             }
             Err(e) => return Err(e),

crates/engine/src/tests.rs

@@ -2817,3 +2817,39 @@ fn oversized_exports_clamp_and_report_their_dpi_unless_strict() {
     let err = export::Exporter::from_source(strict).png(0, 600.0).unwrap_err();
     assert!(err.contains("exceeds renderer limits"), "{err}");
 }
+
+#[test]
+fn legacy_passwords_with_turkish_characters_open() {
+    // RC4-128 (R3): user "ılık€" in PDFDocEncoding (ı = 0x9A, € = 0xA0), owner "Çağlar"
+    // (PDFDocEncoding has no ğ: written without it, as pdfcraft-crypt writes R2–R4 passwords).
+    let mut cos = pdfcraft_cos::Document::open(Arc::new(fixture(1))).unwrap();
+    let params = pdfcraft_cos::NewEncryption {
+        algorithm: pdfcraft_cos::Algorithm::Rc4_128,
+        user_password: "ılık€",
+        owner_password: "Çağlar",
+        permissions: -1,
+        encrypt_metadata: true,
+        seed: [7; 32],
+    };
+    cos.set_encryption(&params).unwrap();
+    let bytes = Arc::new(pdfcraft_cos::write_full(&cos, &pdfcraft_cos::SaveOptions::default()).unwrap());
+    let mut s = Session::new();
+    // The renderer used to try only the UTF-8 bytes, and the owner fallback handed it the
+    // recovered user password as Latin-1: both came back as a wrong password.
+    for pw in ["ılık€", "Çağlar"] {
+        let id = s.open("x.pdf", None, bytes.clone(), Some(pw)).unwrap_or_else(|e| panic!("{pw}: {e:?}"));
+        assert_eq!(page_texts(&s, id), ["Page 1"], "{pw}");
+        assert!(s.get(id).unwrap().info.encrypted, "{pw}");
+    }
+    assert!(matches!(s.open("x.pdf", None, bytes, Some("ilik€")), Err(OpenError::WrongPassword)));
+}
+
+#[test]
+fn recovered_user_passwords_reach_the_renderer_byte_for_byte() {
+    assert_eq!(crate::renderer_password(b"pw"), "pw");
+    // UTF-8 bytes (qpdf writes "şifre" so; its \xC5\x9F has no PDFDocEncoding reading).
+    assert_eq!(crate::renderer_password("şifre".as_bytes()), "şifre");
+    // PDFDocEncoding bytes: ı = 0x9A, € = 0xA0, Ç = 0xC7.
+    assert_eq!(crate::renderer_password(&[0x9A, b'l', 0x9A, b'k', 0xA0]), "ılık€");
+    assert_eq!(crate::renderer_password(&[0xC7, b'o', b'k']), "Çok");
+}

vendor/hayro-syntax/src/crypto/mod.rs

@@ -162,18 +162,27 @@ pub(crate) fn get(
     };
 
     let mut decryption_key = if revision <= 4 {
-        let key = decryption_key_rev1234(
-            password,
-            encrypt_metadata,
-            revision,
-            byte_length,
-            &owner_string,
-            permissions,
-            id,
-        )?;
-        authenticate_user_password_rev234(revision, &key, id, &user_string)?;
-
-        key
+        // PdfCraft patch: (see vendor/README.md hayro-syntax (14)) the password is tried as
+        // given and then in PDFDocEncoding, which revisions 2–4 specify.
+        let mut found = Err(DecryptionError::PasswordProtected);
+        for candidate in legacy_passwords(password) {
+            found = decryption_key_rev1234(
+                &candidate,
+                encrypt_metadata,
+                revision,
+                byte_length,
+                &owner_string,
+                permissions,
+                id,
+            )
+            .and_then(|key| {
+                authenticate_user_password_rev234(revision, &key, id, &user_string).map(|()| key)
+            });
+            if found.is_ok() {
+                break;
+            }
+        }
+        found?
     } else {
         decryption_key_rev56(dict, revision, password, &owner_string, &user_string)?
     };
@@ -448,6 +457,45 @@ fn compute_hash_rev56(
     Ok(result)
 }
 
+/// `PdfCraft` patch: the byte strings to try as a revision 2–4 password. Revisions 2–4 take the
+/// password in `PDFDocEncoding` (ISO 32000-2 §7.6.4.3.2, Algorithm 2 a), but the password arrives
+/// as UTF-8 text: first the bytes as given (what earlier versions used, and what some writers
+/// store), then, for text with characters outside ASCII that `PDFDocEncoding` all has ("ılık€",
+/// "Çok güzel"), its `PDFDocEncoding` bytes.
+fn legacy_passwords(password: &[u8]) -> Vec<Vec<u8>> {
+    let mut out = vec![password.to_vec()];
+    if let Ok(text) = core::str::from_utf8(password)
+        && !text.is_ascii()
+        && let Some(doc) = text.chars().map(pdf_doc_byte).collect::<Option<Vec<u8>>>()
+    {
+        out.push(doc);
+    }
+    out
+}
+
+/// `PdfCraft` patch: `PDFDocEncoding` (ISO 32000-2 Annex D) of `c`, `None` when it has none.
+/// It differs from Latin-1 in 0x18–0x1F and 0x80–0xA0 (0x9F is undefined).
+fn pdf_doc_byte(c: char) -> Option<u8> {
+    const LOW: [char; 8] = ['˘', 'ˇ', 'ˆ', '˙', '˝', '˛', '˚', '˜'];
+    const HIGH: [char; 33] = [
+        '•', '†', '‡', '…', '—', '–', 'ƒ', '⁄', '‹', '›', '−', '‰', '„', '“', '”', '‘', '’', '‚',
+        '™', 'fi', 'fl', 'Ł', 'Œ', 'Š', 'Ÿ', 'Ž', 'ı', 'ł', 'œ', 'š', 'ž', '\u{FFFD}', '€',
+    ];
+    let at = |table: &[char], base: u8| {
+        table
+            .iter()
+            .position(|&t| t == c && t != '\u{FFFD}')
+            .and_then(|i| u8::try_from(i).ok())
+            .map(|i| base + i)
+    };
+    at(&LOW, 0x18)
+        .or_else(|| at(&HIGH, 0x80))
+        .or_else(|| match u32::from(c) {
+            0x18..=0x1F | 0x80..=0xA0 => None,
+            n => u8::try_from(n).ok(),
+        })
+}
+
 /// Algorithm 2: Computing a file encryption key in order to encrypt a document (revision 4 and earlier)
 fn decryption_key_rev1234(
     password: &[u8],
@@ -460,7 +508,7 @@ fn decryption_key_rev1234(
 ) -> Result<Vec<u8>, DecryptionError> {
     let mut md5_input = vec![];
 
-    // TODO: Convert to PDFDocEncoding.
+    // (PDFDocEncoding: see `legacy_passwords`, PdfCraft patch.)
     // a) Pad or truncate password to 32 bytes using PASSWORD_PADDING.
     let mut padded_password = [0_u8; 32];
     let copy_len = password.len().min(32);
@@ -672,3 +720,24 @@ fn decryption_key_rev56(
     // "b". Bytes 0-3 of the decrypted Perms entry, treated as a little-endian integer, are the user
     // permissions. They shall match the value in the P key.
 }
+
+#[cfg(test)]
+mod pdfcraft_password_tests {
+    use super::*;
+
+    #[test]
+    fn legacy_passwords_are_tried_as_given_then_in_pdf_doc_encoding() {
+        assert_eq!(legacy_passwords(b"pw"), [b"pw".to_vec()]);
+        assert_eq!(
+            legacy_passwords("ılık€".as_bytes()),
+            ["ılık€".as_bytes().to_vec(), vec![0x9A, b'l', 0x9A, b'k', 0xA0]]
+        );
+        assert_eq!(
+            legacy_passwords("Çok".as_bytes()),
+            ["Çok".as_bytes().to_vec(), vec![0xC7, b'o', b'k']]
+        );
+        // PDFDocEncoding has no ş: only the bytes as given.
+        assert_eq!(legacy_passwords("şifre".as_bytes()), ["şifre".as_bytes().to_vec()]);
+        assert_eq!(legacy_passwords(&[0xFF, 0x00]), [vec![0xFF, 0x00]]);
+    }
+}