address review: share the first inserted dictionary on a concurrent load

crates/av1/src/decoder.rs

@@ -216,9 +216,6 @@ pub struct Decoder {
     out: VecDeque<Output>,
     /// pts of the temporal unit being decoded.
     pts: i64,
-    /// Bytes of the repeated-frame (`show_existing_frame`) pictures queued by the current
-    /// [`Decoder::decode_pts`] call; each is copied out in full when it is returned.
-    chunk_output_bytes: u64,
 }
 
 /// Whether `pictures` frames of `width` x `height`, padded as `FrameBuf::new` pads them, fit in
@@ -301,7 +298,6 @@ impl Decoder {
             memory_budget: deckcraft_bitstream::budget::DECODE_MEMORY_BUDGET,
             out: VecDeque::new(),
             pts: 0,
-            chunk_output_bytes: 0,
         }
     }
 
@@ -334,7 +330,6 @@ impl Decoder {
     /// [`Decoder::decode`] with a timestamp that the pictures shown by this chunk carry.
     pub fn decode_pts(&mut self, data: &[u8], pts: i64) -> Result<Vec<Picture>> {
         self.pts = pts;
-        self.chunk_output_bytes = 0;
         let parsed = self.parse(data);
         // Hand out what is finished, then report a parse error.
         let mut pics = Vec::new();
@@ -505,20 +500,6 @@ impl Decoder {
             let idx = fh.frame_to_show_map_idx;
             let slot = self.ref_frames[idx].clone().ok_or(Error::Invalid("show_existing_frame of an empty slot"))?;
             let info = self.refs[idx].clone();
-            // Every repeat is copied out whole, so a compact header must not queue more
-            // pictures than half the decoder budget allows.
-            let shape = deckcraft_bitstream::budget::PictureShape {
-                width: u64::from(info.upscaled_width),
-                height: u64::from(info.frame_height),
-                shift_x: u32::from(seq.color.subsampling_x),
-                shift_y: u32::from(seq.color.subsampling_y),
-                chroma_planes: if seq.color.mono_chrome { 0 } else { 2 },
-                bytes_per_sample: 2,
-            };
-            self.chunk_output_bytes = self.chunk_output_bytes.saturating_add(shape.bytes().unwrap_or(u64::MAX));
-            if self.chunk_output_bytes > self.memory_budget / 2 {
-                return Err(Error::Invalid("repeated frame outputs exceed the decoder memory budget"));
-            }
             let o = Output::Existing {
                 slot: slot.clone(),
                 seq: seq.clone(),

crates/engine/src/spell.rs

@@ -171,7 +171,9 @@ impl Dictionary {
                 ["COMPOUNDWORDMAX", n, ..] => d.compound_max = n.parse().unwrap_or(0),
                 ["REP", n] => {
                     for _ in 0..n.parse::<usize>().unwrap_or(0) {
-                        let r: Vec<&str> = lines.next().unwrap_or("").split_whitespace().collect();
+                        // The count is the file's claim: stop when the records run out.
+                        let Some(line) = lines.next() else { break };
+                        let r: Vec<&str> = line.split_whitespace().collect();
                         if let [_, a, b, ..] = r.as_slice() {
                             d.rep.push((a.replace('_', " "), b.replace('_', " ")));
                         }
@@ -182,7 +184,8 @@ impl Dictionary {
                     let Some(flag) = one_flag(Some(flag), mode) else { continue };
                     let cross = cross.eq_ignore_ascii_case("y");
                     for _ in 0..n.parse::<usize>().unwrap_or(0) {
-                        let r: Vec<&str> = lines.next().unwrap_or("").split_whitespace().collect();
+                        let Some(line) = lines.next() else { break };
+                        let r: Vec<&str> = line.split_whitespace().collect();
                         let [_, _, strip, add, rest @ ..] = r.as_slice() else { continue };
                         let (add, cont) = match add.split_once('/') {
                             Some((a, c)) => (a, parse_flags(c, mode)),
@@ -576,6 +579,15 @@ fn cache() -> &'static Cache {
     CACHE.get_or_init(Default::default)
 }
 
+/// Language strings remembered by [`dictionary`]; a string past it is resolved again on each call.
+const MAX_CACHED_LANGS: usize = 256;
+
+/// Parsed dictionaries by their affix file, so aliases of one language share it.
+fn by_files() -> &'static Mutex<HashMap<PathBuf, Arc<Dictionary>>> {
+    static M: OnceLock<Mutex<HashMap<PathBuf, Arc<Dictionary>>>> = OnceLock::new();
+    M.get_or_init(Default::default)
+}
+
 /// The dictionary for `lang` (a BCP-47 tag such as `en-US`), loaded once per process.
 pub fn dictionary(lang: &str) -> Option<Arc<Dictionary>> {
     let cache = cache();
@@ -586,10 +598,23 @@ pub fn dictionary(lang: &str) -> Option<Arc<Dictionary>> {
         return d.clone();
     }
     let loaded = find_in(&search_dirs(), lang).and_then(|(a, d)| {
-        let (a, d) = (std::fs::read(a).ok()?, std::fs::read(d).ok()?);
-        Some(Arc::new(Dictionary::parse(&a, &d)))
+        // Tags that resolve to the same files share one parsed dictionary.
+        if let Ok(m) = by_files().lock()
+            && let Some(dict) = m.get(&a)
+        {
+            return Some(dict.clone());
+        }
+        let (bytes_a, bytes_d) = (std::fs::read(&a).ok()?, std::fs::read(d).ok()?);
+        let dict = Arc::new(Dictionary::parse(&bytes_a, &bytes_d));
+        // A concurrent loader of the same files may have inserted first: share its copy.
+        if let Ok(mut m) = by_files().lock() {
+            return Some(m.entry(a).or_insert(dict).clone());
+        }
+        Some(dict)
     });
-    if let Ok(mut c) = cache.lock() {
+    if let Ok(mut c) = cache.lock()
+        && c.len() < MAX_CACHED_LANGS
+    {
         c.insert(key, loaded.clone());
     }
     loaded
@@ -880,6 +905,14 @@ SFX D Y 1\nSFX D 0 ed/S .\n";
         assert!(d.suggest("workingslides", 5).contains(&"working slides".to_string()));
     }
 
+    /// An affix header can claim billions of records; parsing stops where the file does.
+    #[test]
+    fn a_header_count_past_the_records_ends_with_them() {
+        let aff = b"SFX S Y 4000000000\nSFX S 0 s .\nREP 4000000000\nREP a b\n";
+        let d = Dictionary::parse(aff, b"1\ncat/S\n".as_slice());
+        assert!(d.check("cats"));
+    }
+
     /// #1175: compound splitting is quadratic in the word's length, so a word longer than
     /// `MAX_CHECK_CHARS` is not checked; shorter ones still are.
     #[test]

crates/hevc/src/decoder.rs

@@ -404,9 +404,7 @@ impl Decoder {
             chroma_planes: if chroma == 0 { 0 } else { 2 },
             bytes_per_sample: 2,
         };
-        // Output frames waiting to be returned stay allocated after they leave the DPB.
-        let queued = self.out_queue.iter().filter(|o| !self.dpb.entries.iter().any(|e| Arc::ptr_eq(&e.frame, &o.frame))).count();
-        let pictures = u64::from(sps.max_dec_pic_buffering) + 1 + self.max_in_flight as u64 + queued as u64;
+        let pictures = u64::from(sps.max_dec_pic_buffering) + 1 + self.max_in_flight as u64;
         ensure!(
             shape.fits(pictures, self.memory_budget),
             "{}x{} pictures with {pictures} in the decoder exceed the memory budget",

crates/vp9/src/decoder.rs

@@ -166,17 +166,11 @@ impl Decoder {
         if frames.len() > 1 {
             self.stats.superframes += 1;
         }
-        let queued = self.pending.len();
         for f in frames {
             if f.is_empty() {
                 continue;
             }
-            if let Err(e) = self.decode_frame(f, pts) {
-                // A failed sample outputs nothing: drop what its earlier subframes queued so a
-                // later drain does not expand each repeat into its own picture.
-                self.pending.truncate(queued);
-                return Err(e);
-            }
+            self.decode_frame(f, pts)?;
         }
         Ok(self.drain(false))
     }