Issue #1296: hayro-syntax applies SASLprep to AES-256 passwords (PdfCraft patch 16) (#1345)

Revisions 5-6 of the standard security handler key the file on the
password processed with SASLprep (ISO 32000-2 7.6.4.3.3, Algorithm 2.A
step a). pdfcraft-crypt writes and reads it so, but the vendored
hayro-syntax skipped the step ("TODO: Do the above.") and used the UTF-8
bytes as typed. A password that SASLprep changes (a decomposed "cafe" +
U+0301, a no-break space, fullwidth letters typed by an IME in full-width
mode) was wrong for the renderer: Protect with such an open password was
rolled back ("the password is incorrect"), and an AES-256 file with such a
password did not open in a session with it, although pdfcraft-cos
accepted it.

- hayro-syntax (vendored patch 16) tries an R5/R6 password in its SASLprep
  form (as pdfcraft-crypt computes it: as given if SASLprep refuses it,
  UTF-8, at most 127 bytes), then, when that differs, as given.
  stringprep, already in the tree through pdfcraft-crypt, is an optional
  dependency enabled by the std feature; without std nothing changes.
- Tests: hayro-syntax crypto::pdfcraft_password_tests (candidate list and
  R5/R6 dictionaries built from prepared passwords) and two engine tests
  (Protect and reopen, and pdfcraft-cos-encrypted files) for the three
  cases, with precomposed "caf\u{E9}" as the control. The dictionary test
  and both engine tests fail without the fix.

Fixes #1296.

Cargo.lock

@@ -2372,6 +2372,7 @@ dependencies = [
  "memchr",
  "rustc-hash 2.1.3",
  "smallvec",
+ "stringprep",
  "zune-jpeg",
 ]
 

crates/engine/src/tests.rs

@@ -3775,6 +3775,61 @@ fn recovered_user_passwords_reach_the_renderer_byte_for_byte() {
     assert_eq!(crate::renderer_password(&[0xC7, b'o', b'k']), "Çok");
 }
 
+/// AES-256 passwords as typed, each with its SASLprep form, which ISO 32000-2 keys the file on:
+/// a decomposed é (NFKC composes it), a no-break space (mapped to a space) and fullwidth letters
+/// (what a Japanese IME types in full-width mode; NFKC narrows them). Precomposed "café" is the
+/// control: SASLprep leaves it as it is.
+const SASLPREP_CHANGES: [(&str, &str); 4] =
+    [("cafe\u{301}", "caf\u{E9}"), ("a\u{A0}b", "a b"), ("\u{FF41}\u{FF42}", "ab"), ("caf\u{E9}", "caf\u{E9}")];
+
+#[test]
+fn aes256_open_passwords_that_saslprep_changes_protect_and_reopen_as_typed() {
+    for (typed, prepared) in SASLPREP_CHANGES {
+        let (mut s, id) = session_with(1);
+        // The renderer used to take the password as typed, so the protected file didn't reopen
+        // and the edit was rolled back: "the password is incorrect".
+        s.apply(id, Edit::Protect(protection(Some(typed), None))).unwrap_or_else(|e| panic!("{typed:?}: {e:?}"));
+        assert!(s.get(id).unwrap().info.encrypted, "{typed:?}");
+        assert_eq!(page_texts(&s, id), ["Page 1"], "{typed:?}");
+        let saved = s.save_bytes(id).unwrap();
+        for pw in [typed, prepared] {
+            let mut again = Session::new();
+            let reopened = again.open("x.pdf", None, saved.clone(), Some(pw)).unwrap_or_else(|e| panic!("{typed:?} as {pw:?}: {e:?}"));
+            assert_eq!(page_texts(&again, reopened), ["Page 1"], "{typed:?} as {pw:?}");
+        }
+        assert!(matches!(Session::new().open("x.pdf", None, saved, Some("cafe")), Err(OpenError::WrongPassword)), "{typed:?}");
+    }
+}
+
+#[test]
+fn aes256_files_whose_passwords_saslprep_changes_open_as_typed() {
+    // A fullwidth Ｏ: SASLprep makes the owner password "Owner".
+    let (owner, owner_prepared) = ("\u{FF2F}wner", "Owner");
+    for (typed, prepared) in SASLPREP_CHANGES {
+        // pdfcraft-cos keys the file on the SASLprep forms, as ISO 32000-2 asks.
+        let mut cos = pdfcraft_cos::Document::open(Arc::new(fixture(1))).unwrap();
+        let params = pdfcraft_cos::NewEncryption {
+            algorithm: pdfcraft_cos::Algorithm::Aes256,
+            user_password: typed,
+            owner_password: owner,
+            permissions: -1,
+            encrypt_metadata: true,
+            seed: [7; 32],
+        };
+        cos.set_encryption(&params).unwrap();
+        let bytes = Arc::new(pdfcraft_cos::write_full(&cos, &pdfcraft_cos::SaveOptions::default()).unwrap());
+        let mut s = Session::new();
+        // pdfcraft-cos accepts the password as typed (it applies SASLprep); the renderer used to
+        // refuse it, so the session called it a wrong password.
+        for pw in [typed, prepared, owner, owner_prepared] {
+            let id = s.open("x.pdf", None, bytes.clone(), Some(pw)).unwrap_or_else(|e| panic!("{typed:?} as {pw:?}: {e:?}"));
+            assert_eq!(page_texts(&s, id), ["Page 1"], "{typed:?} as {pw:?}");
+            assert!(s.get(id).unwrap().info.encrypted, "{typed:?} as {pw:?}");
+        }
+        assert!(matches!(s.open("x.pdf", None, bytes, Some("cafe")), Err(OpenError::WrongPassword)), "{typed:?}");
+    }
+}
+
 /// A synthetic phone photo, 400 × 300 px: a white sheet, tilted, on a grey table, with a black
 /// mark in the sheet's top-left corner. Returns the PNG and the sheet's corners in pixels.
 fn camera_photo() -> (Vec<u8>, [[f64; 2]; 4]) {

vendor/hayro-syntax/Cargo.toml

@@ -46,6 +46,7 @@ std = [
     "hayro-jbig2?/std",
     "hayro-jpeg2000?/std",
     "dep:rustc-hash",
+    "dep:stringprep",
     "zune-jpeg?/std",
 ]
 unsafe = [
@@ -102,6 +103,10 @@ optional = true
 [dependencies.smallvec]
 version = "1"
 
+[dependencies.stringprep]
+version = "0.1"
+optional = true
+
 [dependencies.zune-jpeg]
 version = "0.5"
 optional = true

vendor/hayro-syntax/src/crypto/mod.rs

@@ -184,7 +184,16 @@ pub(crate) fn get(
         }
         found?
     } else {
-        decryption_key_rev56(dict, revision, password, &owner_string, &user_string)?
+        // PdfCraft patch: (see vendor/README.md hayro-syntax (16)) the password is tried in its
+        // SASLprep form, which revisions 5–6 specify, and then as given.
+        let mut found = Err(DecryptionError::PasswordProtected);
+        for candidate in aes256_passwords(password) {
+            found = decryption_key_rev56(dict, revision, &candidate, &owner_string, &user_string);
+            if found.is_ok() {
+                break;
+            }
+        }
+        found?
     };
 
     // See pdf.js issue 19484.
@@ -496,6 +505,38 @@ fn pdf_doc_byte(c: char) -> Option<u8> {
         })
 }
 
+/// `PdfCraft` patch: the byte strings to try as a revision 5–6 password. Revisions 5–6 take the
+/// password processed with `SASLprep` (RFC 4013) and in UTF-8, at most 127 bytes (ISO 32000-2
+/// §7.6.4.3.3, Algorithm 2.A a–b), but the password arrives as typed: first its `SASLprep`
+/// form, as pdfcraft-crypt writes it (a decomposed "café" composed, a fullwidth "ａｂ" narrowed,
+/// a no-break space made a space), then, when that differs, the bytes as given (what earlier
+/// versions used, and what writers that skip `SASLprep` store), each at most 127 bytes.
+fn aes256_passwords(password: &[u8]) -> Vec<Vec<u8>> {
+    let given = password.get(..127).unwrap_or(password).to_vec();
+    match sasl_prepared(password) {
+        Some(prepared) if prepared != given => vec![prepared, given],
+        _ => vec![given],
+    }
+}
+
+/// `PdfCraft` patch: `password` processed with `SASLprep`, in UTF-8 and truncated to 127 bytes,
+/// as pdfcraft-crypt does it: a password `SASLprep` refuses (one with prohibited or unassigned
+/// characters) is taken as given. `None` when `password` isn't UTF-8.
+#[cfg(feature = "std")]
+fn sasl_prepared(password: &[u8]) -> Option<Vec<u8>> {
+    let text = core::str::from_utf8(password).ok()?;
+    let prepared = stringprep::saslprep(text).map_or_else(|_| text.to_string(), |p| p.into_owned());
+    let mut bytes = prepared.into_bytes();
+    bytes.truncate(127);
+    Some(bytes)
+}
+
+/// `PdfCraft` patch: without `std`, which stringprep needs, passwords are taken as given.
+#[cfg(not(feature = "std"))]
+fn sasl_prepared(_: &[u8]) -> Option<Vec<u8>> {
+    None
+}
+
 /// Algorithm 2: Computing a file encryption key in order to encrypt a document (revision 4 and earlier)
 fn decryption_key_rev1234(
     password: &[u8],
@@ -648,7 +689,7 @@ fn decryption_key_rev56(
     // a) The UTF-8 password string shall be generated from Unicode input by processing the input string with
     // the SASLprep (Internet RFC 4013) profile of stringprep (Internet RFC 3454) using the Normalize and BiDi
     // options, and then converting to a UTF-8 representation.
-    // TODO: Do the above.
+    // (SASLprep: see `aes256_passwords`, PdfCraft patch.)
     // b) Truncate the UTF-8 representation to 127 bytes if it is longer than 127 bytes.
     let password = &password[..password.len().min(127)];
 
@@ -740,4 +781,125 @@ mod pdfcraft_password_tests {
         assert_eq!(legacy_passwords("şifre".as_bytes()), ["şifre".as_bytes().to_vec()]);
         assert_eq!(legacy_passwords(&[0xFF, 0x00]), [vec![0xFF, 0x00]]);
     }
+
+    #[test]
+    #[cfg(feature = "std")]
+    fn aes256_passwords_are_tried_in_their_saslprep_form_then_as_given() {
+        let both = |prepared: &str, given: &str| [prepared.into(), given.as_bytes().to_vec()];
+        assert_eq!(aes256_passwords(b"pw"), [b"pw".to_vec()]);
+        assert_eq!(
+            aes256_passwords("caf\u{E9}".as_bytes()),
+            ["caf\u{E9}".as_bytes().to_vec()]
+        );
+        assert_eq!(
+            aes256_passwords("cafe\u{301}".as_bytes()),
+            both("caf\u{E9}", "cafe\u{301}")
+        );
+        assert_eq!(
+            aes256_passwords("a\u{A0}b".as_bytes()),
+            both("a b", "a\u{A0}b")
+        );
+        assert_eq!(
+            aes256_passwords("\u{FF41}\u{FF42}".as_bytes()),
+            both("ab", "\u{FF41}\u{FF42}")
+        );
+        // A soft hyphen is mapped to nothing.
+        assert_eq!(
+            aes256_passwords("pass\u{AD}word".as_bytes()),
+            both("password", "pass\u{AD}word")
+        );
+        // SASLprep refuses a control character: as given.
+        assert_eq!(aes256_passwords(b"a\x07b"), [b"a\x07b".to_vec()]);
+        // Not UTF-8: as given.
+        assert_eq!(aes256_passwords(&[0xFF, 0x00]), [vec![0xFF, 0x00]]);
+        // At most 127 bytes, cut after SASLprep: 64 decomposed é (192 bytes) become 128 bytes of
+        // precomposed é, cut inside the last one.
+        let long = "e\u{301}".repeat(64);
+        let prepared = "\u{E9}".repeat(64);
+        assert_eq!(
+            aes256_passwords(long.as_bytes()),
+            [
+                prepared.as_bytes()[..127].to_vec(),
+                long.as_bytes()[..127].to_vec()
+            ]
+        );
+    }
+
+    /// A revision 5 or 6 `/Encrypt` dictionary whose user and owner passwords are the bytes
+    /// `user` and `owner` and whose file key is `key` (ISO 32000-2 Algorithms 8 and 9).
+    #[cfg(feature = "std")]
+    fn rev56_dict(revision: u8, user: &[u8], owner: &[u8], key: &[u8; 32]) -> Vec<u8> {
+        let hash = |password: &[u8], salt: &[u8], u: Option<&[u8]>| {
+            compute_hash_rev56(password, salt, u, revision).unwrap()
+        };
+        // AES-256 in CBC mode without padding: `encrypt_cbc` pads, which only adds a block.
+        let wrap =
+            |k: [u8; 32]| AES256Cipher::new(&k).unwrap().encrypt_cbc(key, &[0; 16])[..32].to_vec();
+        let (user_validation, user_key, owner_validation, owner_key) =
+            ([1; 8], [2; 8], [3; 8], [4; 8]);
+        let u = [
+            &hash(user, &user_validation, None)[..],
+            &user_validation,
+            &user_key,
+        ]
+        .concat();
+        let ue = wrap(hash(user, &user_key, None));
+        let o_hash = hash(owner, &owner_validation, Some(&u));
+        let o = [&o_hash[..], &owner_validation, &owner_key].concat();
+        let oe = wrap(hash(owner, &owner_key, Some(&u)));
+        let hex = |b: &[u8]| b.iter().map(|b| format!("{b:02X}")).collect::<String>();
+        let (o, u, oe, ue) = (hex(&o), hex(&u), hex(&oe), hex(&ue));
+        format!(
+            "<< /Filter /Standard /V 5 /R {revision} /Length 256 /P -4 \
+             /CF << /StdCF << /CFM /AESV3 /Length 32 >> >> /StmF /StdCF /StrF /StdCF \
+             /O <{o}> /U <{u}> /OE <{oe}> /UE <{ue}> >>"
+        )
+        .into_bytes()
+    }
+
+    /// The file key that `password` unlocks in the `/Encrypt` dictionary `dict`.
+    #[cfg(feature = "std")]
+    fn unlock(dict: &[u8], password: &str) -> Result<Vec<u8>, DecryptionError> {
+        use crate::reader::{Reader, ReaderContext, ReaderExt};
+        let dict = Reader::new(dict).read_with_context::<Dict<'_>>(&ReaderContext::dummy());
+        match get(&dict.unwrap(), b"", password.as_bytes())? {
+            Decryptor::Aes256 { key, .. } => Ok(key),
+            other => panic!("not AES-256: {other:?}"),
+        }
+    }
+
+    #[test]
+    #[cfg(feature = "std")]
+    fn aes256_passwords_open_as_typed_and_in_their_saslprep_form() {
+        let key = [9; 32];
+        // As typed, and the SASLprep form that ISO 32000-2 keys the file on (and pdfcraft-crypt
+        // writes): a decomposed é (NFKC composes it), a no-break space (mapped to a space) and
+        // fullwidth letters (what a Japanese IME types in full-width mode; NFKC narrows them).
+        // Precomposed "café" is the control: SASLprep leaves it as it is.
+        let cases = [
+            ("cafe\u{301}", "caf\u{E9}"),
+            ("a\u{A0}b", "a b"),
+            ("\u{FF41}\u{FF42}", "ab"),
+            ("caf\u{E9}", "caf\u{E9}"),
+        ];
+        let (ok, wrong) = (Ok(key.to_vec()), Err(DecryptionError::PasswordProtected));
+        // Revision 5 hashes once, so it takes every case; revision 6, whose hash is far slower
+        // in a debug build, takes the first.
+        for (typed, prepared) in cases {
+            let user = rev56_dict(5, prepared.as_bytes(), b"owner", &key);
+            let owner = rev56_dict(5, b"user", prepared.as_bytes(), &key);
+            for pw in [typed, prepared] {
+                assert_eq!(unlock(&user, pw), ok, "{typed:?}: user {pw:?}");
+                assert_eq!(unlock(&owner, pw), ok, "{typed:?}: owner {pw:?}");
+            }
+            assert_eq!(unlock(&user, "cafe"), wrong, "{typed:?}");
+            // A writer that skipped SASLprep keyed the file on the password as typed: it opens
+            // with that.
+            let unprepared = rev56_dict(5, typed.as_bytes(), b"owner", &key);
+            assert_eq!(unlock(&unprepared, typed), ok, "{typed:?}: unprepared");
+        }
+        let user = rev56_dict(6, "caf\u{E9}".as_bytes(), b"owner", &key);
+        assert_eq!(unlock(&user, "cafe\u{301}"), ok, "R6");
+        assert_eq!(unlock(&user, "cafe"), wrong, "R6");
+    }
 }