fix(cos): a nonpositive crypt-filter /Length no longer overflows (#1816) (#1864)

encrypt_dict multiplied any V4 StdCF /Length up to 32 by 8, so a large
negative value overflowed (a panic with overflow checks, a wrapped length
without). A zero or negative length is meaningless; read it as if absent, the
128-bit default, which also fixes small negatives that used to clamp to a
40-bit key and fail to open.


Claude-Session: https://claude.ai/code/session_01RUNQSPVXx3dDBabRZHAnjz

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

crates/cos/src/security.rs

@@ -34,12 +34,14 @@ pub(crate) fn encrypt_dict(doc: &Document, d: &Dict) -> EncryptDict {
     }
     let v = int(b"V", 0);
     // V4: the key length comes from the standard crypt filter's /Length (bytes, or bits in
-    // some files); /Length on the dictionary is often a stale 40.
+    // some files); /Length on the dictionary is often a stale 40. A length that isn't
+    // positive means nothing and reads as if absent.
     let cf_len = d
         .get(b"CF")
         .map(|c| doc.resolve(c))
         .and_then(|c| c.as_dict().and_then(|c| c.get(b"StdCF").map(|f| doc.resolve(f))))
-        .and_then(|f| f.as_dict().and_then(|f| f.int(b"Length")));
+        .and_then(|f| f.as_dict().and_then(|f| f.int(b"Length")))
+        .filter(|n| *n > 0);
     let length_bits = match (v, cf_len) {
         (5.., _) => 256,
         (4, Some(n)) if n <= 32 => n * 8,

crates/cos/tests/encryption.rs

@@ -278,3 +278,26 @@ fn signature_contents_stay_unencrypted() {
         assert_eq!(string(2, b"Contents"), b"Private note", "{alg:?}");
     }
 }
+
+#[test]
+fn a_nonpositive_crypt_filter_length_reads_as_the_default_key_length() {
+    // A V4 crypt filter's /Length counts bytes (≤ 32) or bits. A zero or negative value is
+    // meaningless and is read as if absent (128 bits), not multiplied into an overflow (#1816).
+    let original = protect(Algorithm::Aes128, "", "owner", -1);
+    for length in [i64::MIN, i64::MIN / 8 - 1, i64::MIN / 8, -1, 0, 16, 128] {
+        let mut doc = Document::open(Arc::new(original.clone())).unwrap();
+        let encrypt = doc.trailer().reference(b"Encrypt").unwrap();
+        doc.update_dict(encrypt, |e| {
+            let Some(Object::Dict(cf)) = e.get(b"CF") else { panic!("no /CF") };
+            let Some(Object::Dict(std_cf)) = cf.get(b"StdCF") else { panic!("no /StdCF") };
+            let (mut cf, mut std_cf) = (cf.clone(), std_cf.clone());
+            std_cf.set(b"Length".to_vec(), Object::Int(length));
+            cf.set(b"StdCF".to_vec(), Object::Dict(std_cf));
+            e.set(b"CF".to_vec(), Object::Dict(cf));
+        })
+        .unwrap();
+        let saved = write_incremental(&doc, &SaveOptions::default()).unwrap();
+        let again = Document::open(Arc::new(saved)).unwrap();
+        assert_eq!(content(&again), SECRET, "/Length {length}");
+    }
+}