PDF import: the encryption pre-scan reads a bounded window after each trailer (#2142)

The pre-scan lexed one object after every `trailer` marker over the rest of
the file, so an unterminated string or dictionary after each of many markers
made opening a crafted PDF quadratic. Each lex now gets a 4 KiB window; when
it runs out, the file is refused if `/Encrypt` appears past the window (one
search per file), so long trailers can't hide the key.

crates/pdf/src/read.rs

@@ -2,6 +2,7 @@
 //! Only reachable objects are retained, references keep their generation, and native parser
 //! panics (including lazy object reads) discard the entire partially populated adapter.
 
+use std::cell::LazyCell;
 use std::collections::{HashMap, HashSet, VecDeque};
 use std::panic::{AssertUnwindSafe, catch_unwind};
 use std::sync::Arc;
@@ -16,6 +17,14 @@ const MAX_OBJECTS: usize = 100_000;
 const MAX_VALUES: usize = 1_000_000;
 const MAX_DEPTH: usize = 64;
 const MAX_BYTES: usize = 512 * 1024 * 1024;
+/// How far after each `trailer` marker the encryption pre-scan lexes. A classic trailer
+/// dictionary is a few hundred bytes. If the lexer reaches the window's end, the dictionary
+/// may start or continue past it, so the file is refused when `/Encrypt` appears anywhere past
+/// the window. This fallback is conservative: it refuses on any `/Encrypt` text there, which
+/// in practice only encrypted files contain. The pre-scan stays linear in file size, at most
+/// `MAX_OBJECTS` windows (about 400 MB) and one search for `/Encrypt`, even when a marker is
+/// followed by an unterminated string or dictionary the lexer would follow to the end.
+const TRAILER_WINDOW: usize = 4 * 1024;
 
 fn limit(what: &str) -> Error {
     Error::Read(format!("PDF exceeds the {what} import limit"))
@@ -41,6 +50,7 @@ fn open_inner(bytes: &[u8]) -> Result<File, Error> {
     // Preserve the app's refusal of encrypted PDFs, including empty-user-password files.
     // Classic trailer dictionaries use the existing lexer; xref-stream dictionaries below
     // are read by the shared parser. This does not parse document objects a second time.
+    let last_encrypt = LazyCell::new(|| bytes.windows(8).rposition(|word| word == b"/Encrypt"));
     let mut markers = 0;
     for (offset, word) in bytes.windows(7).enumerate() {
         if word == b"trailer" {
@@ -48,9 +58,15 @@ fn open_inner(bytes: &[u8]) -> Result<File, Error> {
             if markers > MAX_OBJECTS {
                 return Err(limit("trailer count"));
             }
-            if let Some(Obj::Dict(dict)) = Lexer::new(bytes, offset + 7).next()
-                && dict.contains_key("Encrypt")
-            {
+            let start = offset + 7;
+            let end = bytes.len().min(start.saturating_add(TRAILER_WINDOW));
+            let window = bytes.get(start..end).unwrap_or_default();
+            let mut lexer = Lexer::new(window, 0);
+            let encrypt = matches!(lexer.next(), Some(Obj::Dict(dict)) if dict.contains_key("Encrypt"));
+            // Whatever it returned, a lexer at the window's end (one past it after an unterminated
+            // hex string) may have stopped before the dictionary started or partway through it.
+            let cut = lexer.pos >= window.len() && last_encrypt.is_some_and(|at| at.saturating_add(8) > end);
+            if encrypt || cut {
                 return Err(Error::Encrypted);
             }
         }

crates/pdf/src/tests/reader.rs

@@ -74,3 +74,51 @@ fn duplicate_unreferenced_compressed_objects_do_not_block_repair() {
     assert!(doc.skipped.iter().any(|warning| warning.contains("recovered PDF object from an object stream: 10 0")));
     assert!(matches!(parse(&document("/Missing 30 0 R", &objects)), Err(Error::Read(message)) if message.contains("ambiguous recovered object 30")));
 }
+
+#[test]
+fn many_trailer_markers_with_unterminated_objects_do_not_rescan_the_file() {
+    // The encryption pre-scan lexes after every `trailer` marker. Unbounded, each unterminated
+    // string or dictionary here ran to the end of the file: about 10^11 bytes lexed (minutes),
+    // against 2 × 10^8 within the window. The markers sit in an unreferenced stream, so the
+    // file still opens as the plain document.
+    let junk = "trailer(trailer<<".repeat(25_000) + &" ".repeat(2 << 20);
+    let stream = (6, 0, format!("<< /Length {} >>\nstream\n{junk}\nendstream", junk.len()));
+    let plain = parse(&document("", &[])).unwrap();
+    assert_eq!(parse(&document("", &[stream])), Ok(plain));
+}
+
+#[test]
+fn a_classic_trailer_with_an_encrypt_entry_is_refused() {
+    let plain = String::from_utf8(document("", &[])).unwrap();
+    let encrypted = plain.replace("/Root 1 0 R >>", "/Root 1 0 R /Encrypt 9 0 R >>");
+    assert_ne!(encrypted, plain);
+    // Object 9 does not exist: the trailer pre-scan refuses the file before the shared parser runs.
+    assert_eq!(parse(encrypted.as_bytes()), Err(Error::Encrypted));
+}
+
+#[test]
+fn an_encrypt_entry_past_the_trailer_window_is_refused() {
+    // The filler pushes /Encrypt past the pre-scan's 4 KiB window. Object 9 does not exist, and the
+    // shared parser reads /Encrypt only as a dictionary, so it alone would open the file.
+    let plain = String::from_utf8(document("", &[])).unwrap();
+    let filler = "x".repeat(8 * 1024);
+    let long = plain.replace("/Root 1 0 R >>", &format!("/Root 1 0 R /Filler ({filler}) >>"));
+    let encrypted = plain.replace("/Root 1 0 R >>", &format!("/Root 1 0 R /Filler ({filler}) /Encrypt 9 0 R >>"));
+    let doc = parse(plain.as_bytes()).unwrap();
+    assert_eq!(parse(long.as_bytes()), Ok(doc));
+    assert_eq!(parse(encrypted.as_bytes()), Err(Error::Encrypted));
+}
+
+#[test]
+fn an_encrypt_entry_after_a_window_cut_before_the_trailer_dictionary_is_refused() {
+    // The 4 KiB pre-scan window ends in the whitespace before the trailer dictionary (the lexer
+    // returns nothing) or between the two `<` of its `<<` (the lexer reads a hex string).
+    let plain = String::from_utf8(document("", &[])).unwrap();
+    let doc = parse(plain.as_bytes()).unwrap();
+    for pad in [4 * 1024, 4 * 1024 - 2] {
+        let spaced = plain.replace("trailer\n<<", &format!("trailer\n{}<<", " ".repeat(pad)));
+        let encrypted = spaced.replace("/Root 1 0 R >>", "/Root 1 0 R /Encrypt 9 0 R >>");
+        assert_eq!(parse(spaced.as_bytes()).as_ref(), Ok(&doc));
+        assert_eq!(parse(encrypted.as_bytes()), Err(Error::Encrypted));
+    }
+}