security(hevc): validate slice payload before allocation (#2939)

crates/hevc/src/decoder.rs

@@ -340,6 +340,9 @@ impl Decoder {
         )?;
         sps.check_supported()?;
         pps.check_supported()?;
+        // `start_picture` creates the frame and DPB state. Do not allocate those for a NAL that
+        // contains only a syntactically valid slice header and no CABAC payload.
+        ensure!(sh.data_offset < rbsp.len(), "empty slice segment data");
         if first {
             self.start_picture(&sh, &sps, &pps, pts)?;
         } else {

crates/hevc/src/params.rs

@@ -6,6 +6,16 @@ use crate::spec_tables::{DEFAULT_SCALING_INTER, DEFAULT_SCALING_INTRA};
 use crate::tables::scan_diag;
 use filmcraft_bitstream::BitReader;
 
+/// HEVC picture state includes several per-pixel and per-block work buffers. This permits 8K
+/// pictures while rejecting dimensions that could exhaust memory during decode.
+pub const MAX_LUMA_SAMPLES: u64 = 1 << 26;
+
+fn check_picture_size(width: u32, height: u32) -> Result<()> {
+    ensure!(width > 0 && height > 0 && width <= 16888 && height <= 16888, "picture size {width}x{height} out of range");
+    ensure!(u64::from(width) * u64::from(height) <= MAX_LUMA_SAMPLES, "picture size {width}x{height} exceeds the decoded-frame budget");
+    Ok(())
+}
+
 /// General profile / tier / level information.
 #[derive(Clone, Debug, Default, PartialEq, Eq)]
 pub struct ProfileTierLevel {
@@ -478,7 +488,7 @@ impl Sps {
         let separate_colour_plane = if chroma_format_idc == 3 { r.read_flag()? } else { false };
         let width = r.read_ue()?;
         let height = r.read_ue()?;
-        ensure!(width > 0 && height > 0 && width <= 16888 && height <= 16888, "picture size {width}x{height} out of range");
+        check_picture_size(width, height)?;
         let mut conf_win = (0, 0, 0, 0);
         if r.read_flag()? {
             conf_win = (r.read_ue()?, r.read_ue()?, r.read_ue()?, r.read_ue()?);
@@ -921,3 +931,15 @@ impl Layout {
         self.tile_id[self.rs_to_ts[rs as usize] as usize]
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn picture_size_has_an_aggregate_pixel_budget() {
+        check_picture_size(8192, 8192).unwrap();
+        let err = check_picture_size(8192, 8193).unwrap_err();
+        assert!(err.to_string().contains("decoded-frame budget"));
+    }
+}