Compare commits

..
Author SHA1 Message Date
Dylan M. Taylor 001e6a5063 H.264: bound decoder work from overlapping slices, gaps and queued output
- A macroblock that an earlier slice of the same picture already decoded is
  refused, so overlapping slices cannot redo the picture's reconstruction
  once per slice.
- The memory budget check counts pictures waiting in the output queue that
  have left the DPB (one decode call may carry many access units) and the
  DPB's actual size, counting shared buffers once.
- frame_num gap concealment advances the POC state across the skipped run in
  constant time instead of once per missing frame_num.
- Storing a reference picture evicts until the DPB is under capacity, rather
  than evicting one picture and inserting regardless.
2026-11-02 22:12:19 -05:00
7 changed files with 141 additions and 165 deletions
+1 -6
View File
@@ -227,9 +227,6 @@ impl FontDb {
}
}
/// Most caret records read for one ligature glyph.
const MAX_LIGATURE_CARETS: usize = 64;
/// Caret positions inside a ligature glyph from GDEF `LigCaretList`, in font units from the
/// glyph's origin (sorted ascending). Empty when the font has none for `gid`.
pub fn ligature_carets(face: &FontFace, gid: u32) -> Vec<i32> {
@@ -239,12 +236,9 @@ pub fn ligature_carets(face: &FontFace, gid: u32) -> Vec<i32> {
let Ok(cov) = list.coverage() else { return Vec::new() };
let Some(idx) = cov.get(skrifa::GlyphId::new(gid)) else { return Vec::new() };
let Ok(lig) = list.lig_glyphs().get(idx as usize) else { return Vec::new() };
// The limit applies to the records read, not to the carets kept: a font can list thousands of
// records it then has to be read through for every ligature laid out.
let mut out: Vec<i32> = lig
.caret_values()
.iter()
.take(MAX_LIGATURE_CARETS)
.filter_map(|cv| cv.ok())
.filter_map(|cv| match cv {
skrifa::raw::tables::gdef::CaretValue::Format1(f) => Some(f.coordinate() as i32),
@@ -252,6 +246,7 @@ pub fn ligature_carets(face: &FontFace, gid: u32) -> Vec<i32> {
// Format 2 is a contour point index; outlines aren't consulted here.
skrifa::raw::tables::gdef::CaretValue::Format2(_) => None,
})
.take(64)
.collect();
out.sort_unstable();
out
+17 -142
View File
@@ -368,7 +368,7 @@ struct CatalogEntry {
/// Process-wide font database.
pub struct FontDb {
faces: RwLock<Vec<Arc<FontFace>>>,
outlines: Mutex<OutlineCache>,
outlines: Mutex<HashMap<(u32, u32), Arc<BezPath>>>,
#[cfg(not(target_arch = "wasm32"))]
catalog: RwLock<Vec<CatalogEntry>>,
/// The folders the system font scan reads (the platform's font folders for [`FontDb::global`]).
@@ -439,38 +439,6 @@ const SYSTEM_FALLBACKS: &[&str] = &[
static NEXT_ID: AtomicU32 = AtomicU32::new(1);
const OUTLINE_CACHE_MAX: usize = 50_000;
/// Most path elements one glyph outline may expand to. Composite and `VARC` glyphs can draw a
/// shared outline many times over; a glyph past this draws nothing rather than holding megabytes.
const MAX_OUTLINE_ELEMENTS: usize = 1 << 15;
/// Most path elements the outline cache holds across all glyphs (about 230 MB at most) before it
/// starts over: distinct glyph ids can each expand one dense outline.
const OUTLINE_CACHE_MAX_ELEMENTS: usize = 1 << 22;
/// Glyph outlines by (face, glyph), with the path elements they hold.
#[derive(Default)]
struct OutlineCache {
paths: HashMap<(u32, u32), Arc<BezPath>>,
elements: usize,
}
impl OutlineCache {
fn get(&self, key: &(u32, u32)) -> Option<Arc<BezPath>> {
self.paths.get(key).cloned()
}
fn insert(&mut self, key: (u32, u32), path: Arc<BezPath>) {
let n = path.elements().len();
if let Some(old) = self.paths.remove(&key) {
self.elements = self.elements.saturating_sub(old.elements().len());
}
if self.paths.len() >= OUTLINE_CACHE_MAX || self.elements.saturating_add(n) > OUTLINE_CACHE_MAX_ELEMENTS {
self.paths.clear();
self.elements = 0;
}
self.paths.insert(key, path);
self.elements = self.elements.saturating_add(n);
}
}
fn name(font: &skrifa::FontRef<'_>, ids: &[StringId]) -> Option<String> {
ids.iter().find_map(|id| font.localized_strings(*id).english_or_first().map(|s| s.to_string()).filter(|s| !s.is_empty()))
@@ -726,10 +694,6 @@ const MAX_COLLECTION_FACES: u32 = 256;
const MAX_COLLECTION_ENTRIES_SCANNED: u32 = 16_384;
/// Maximum named instances read from a single face.
const MAX_NAMED_INSTANCES_PER_FACE: usize = 1024;
/// Maximum faces (named instances counted separately) enumerated from one font file, across all
/// of a collection's faces: the per-collection and per-face limits multiply, and each face found
/// is then built with its own shaper.
const MAX_ENUMERATED_FACES: usize = 4096;
fn distinct_face_indices(entries: impl IntoIterator<Item = (u32, u32)>) -> Vec<u32> {
let mut offsets = std::collections::HashSet::new();
@@ -766,30 +730,21 @@ fn enumerate_faces(data: &[u8]) -> Vec<Found> {
Err(_) => Vec::new(),
};
let mut out = Vec::new();
// Named instances read and plain faces kept, across the whole file.
let mut budget = MAX_ENUMERATED_FACES;
for i in indices {
if budget == 0 {
break;
}
let Ok(f) = skrifa::FontRef::from_index(data, i) else { continue };
let Some((family, style)) = face_names(&f) else { continue };
let axes = f.axes();
let mut named = Vec::new();
let mut seen = std::collections::HashSet::new();
let instances = f.named_instances();
for ni in instances.iter().take(capped_named_instance_count(instances.len())) {
let Some(left) = budget.checked_sub(1) else { break };
budget = left;
let Some(style) = name(&f, &[ni.subfamily_name_id()]) else { continue };
if !seen.insert(style.to_ascii_lowercase()) {
if named.iter().any(|(_, s, _): &(u32, String, _)| s.eq_ignore_ascii_case(&style)) {
continue;
}
let coords: Vec<([u8; 4], f32)> = axes.iter().zip(ni.user_coords()).map(|(a, v)| (a.tag().to_be_bytes(), v)).collect();
named.push((i, style, coords));
}
if named.is_empty() {
budget = budget.saturating_sub(1);
out.push((i, family, style, Vec::new()));
} else {
out.extend(named.into_iter().map(|(i, s, c)| (i, family.clone(), s, c)));
@@ -950,7 +905,7 @@ impl FontDb {
}
Self {
faces: RwLock::new(faces),
outlines: Mutex::new(OutlineCache::default()),
outlines: Mutex::new(HashMap::new()),
#[cfg(not(target_arch = "wasm32"))]
catalog: RwLock::new(Vec::new()),
font_dirs,
@@ -1558,72 +1513,41 @@ impl FontDb {
pub fn outline(&self, face: &FontFace, gid: u32) -> Arc<BezPath> {
let key = (face.id, gid);
if let Some(p) = self.outlines.lock().unwrap_or_else(std::sync::PoisonError::into_inner).get(&key) {
return p;
return p.clone();
}
let mut pen = FlipPen::new(MAX_OUTLINE_ELEMENTS);
let mut pen = FlipPen(BezPath::new());
if let Some(f) = face.skrifa()
&& let Some(g) = f.outline_glyphs().get(GlyphId::new(gid))
{
let _ = g.draw(DrawSettings::unhinted(Size::unscaled(), face.location()), &mut pen);
}
let p = Arc::new(pen.finish());
self.outlines.lock().unwrap_or_else(std::sync::PoisonError::into_inner).insert(key, p.clone());
let p = Arc::new(pen.0);
let mut cache = self.outlines.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if cache.len() >= OUTLINE_CACHE_MAX {
cache.clear();
}
cache.insert(key, p.clone());
p
}
}
/// Collects a glyph outline, y flipped, up to `limit` path elements. Past the limit it ignores
/// what follows and [`FlipPen::finish`] gives an empty path: a cut-off outline would fill wrongly.
struct FlipPen {
path: BezPath,
limit: usize,
over: bool,
}
impl FlipPen {
fn new(limit: usize) -> Self {
FlipPen { path: BezPath::new(), limit, over: false }
}
/// Room for one more element?
fn room(&mut self) -> bool {
if !self.over && self.path.elements().len() >= self.limit {
self.over = true;
self.path = BezPath::new();
}
!self.over
}
fn finish(self) -> BezPath {
if self.over { BezPath::new() } else { self.path }
}
}
struct FlipPen(BezPath);
impl OutlinePen for FlipPen {
fn move_to(&mut self, x: f32, y: f32) {
if self.room() {
self.path.move_to((x as f64, -y as f64));
}
self.0.move_to((x as f64, -y as f64));
}
fn line_to(&mut self, x: f32, y: f32) {
if self.room() {
self.path.line_to((x as f64, -y as f64));
}
self.0.line_to((x as f64, -y as f64));
}
fn quad_to(&mut self, cx0: f32, cy0: f32, x: f32, y: f32) {
if self.room() {
self.path.quad_to((cx0 as f64, -cy0 as f64), (x as f64, -y as f64));
}
self.0.quad_to((cx0 as f64, -cy0 as f64), (x as f64, -y as f64));
}
fn curve_to(&mut self, cx0: f32, cy0: f32, cx1: f32, cy1: f32, x: f32, y: f32) {
if self.room() {
self.path.curve_to((cx0 as f64, -cy0 as f64), (cx1 as f64, -cy1 as f64), (x as f64, -y as f64));
}
self.0.curve_to((cx0 as f64, -cy0 as f64), (cx1 as f64, -cy1 as f64), (x as f64, -y as f64));
}
fn close(&mut self) {
if self.room() {
self.path.close_path();
}
self.0.close_path();
}
}
@@ -1651,55 +1575,6 @@ mod font_dir_tests {
assert_eq!(capped_named_instance_count(MAX_NAMED_INSTANCES_PER_FACE + 1), MAX_NAMED_INSTANCES_PER_FACE);
}
#[test]
fn faces_of_one_file_are_bounded_across_the_collection() {
// The per-collection and per-face limits together would allow far more.
assert!((MAX_COLLECTION_FACES as usize) * MAX_NAMED_INSTANCES_PER_FACE > 16 * MAX_ENUMERATED_FACES);
for data in bundled() {
assert!(enumerate_faces(data).len() <= MAX_ENUMERATED_FACES);
}
}
#[test]
fn an_outline_past_the_element_limit_draws_nothing() {
let mut pen = FlipPen::new(8);
pen.move_to(0.0, 0.0);
pen.line_to(1.0, 1.0);
pen.close();
assert_eq!(pen.finish().elements().len(), 3);
// A shared outline drawn over and over (composite and VARC glyphs).
let mut pen = FlipPen::new(8);
for i in 0..1_000_000 {
pen.move_to(0.0, 0.0);
pen.line_to(i as f32, 1.0);
pen.close();
}
assert_eq!(pen.path.elements().len(), 0);
assert!(pen.finish().elements().is_empty());
}
#[test]
fn outline_cache_is_bounded_by_elements() {
let mut path = BezPath::new();
path.move_to((0.0, 0.0));
for i in 1..MAX_OUTLINE_ELEMENTS {
path.line_to((i as f64, 0.0));
}
let path = Arc::new(path);
let mut cache = OutlineCache::default();
let glyphs = 4 * OUTLINE_CACHE_MAX_ELEMENTS / MAX_OUTLINE_ELEMENTS;
for gid in 0..glyphs as u32 {
cache.insert((1, gid), path.clone());
assert!(cache.elements <= OUTLINE_CACHE_MAX_ELEMENTS);
assert_eq!(cache.elements, cache.paths.values().map(|p| p.elements().len()).sum::<usize>());
}
assert!(cache.get(&(1, glyphs as u32 - 1)).is_some());
// Replacing an entry does not count it twice.
let before = cache.elements;
cache.insert((1, glyphs as u32 - 1), path.clone());
assert_eq!(cache.elements, before);
}
#[test]
fn xdg_data_dirs_contribute_their_fonts_folders() {
let home = Path::new("/home/ann");
+1 -1
View File
@@ -48,7 +48,7 @@ impl SliceDecoder<'_> {
let intra_slice = self.sh.slice_type.is_intra();
loop {
ensure!(addr < total, "macroblock address {addr} out of range");
self.start_mb(addr);
self.start_mb(addr)?;
let skip = !intra_slice && self.decode_skip_flag(&mut c);
if skip {
self.decode_skip()?;
+5 -1
View File
@@ -451,7 +451,11 @@ impl Decoder {
chroma_planes: if chroma == 0 { 0 } else { 2 },
bytes_per_sample: 1,
};
let pictures = (sps.max_dpb_frames() + 1 + self.max_in_flight) as u64;
// Pictures waiting for output that already left the DPB are held too (one decode call may
// carry many access units). A buffer shared with a DPB entry is counted once.
let queued = self.out_queue.iter().filter(|o| !self.dpb.entries.iter().any(|e| Arc::ptr_eq(&e.frame, &o.frame))).count();
let dpb = sps.max_dpb_frames().max(self.dpb.entries.len());
let pictures = (dpb + 1 + self.max_in_flight.max(queued)) as u64;
ensure!(
shape.fits(pictures, self.memory_budget),
"{}x{} pictures with {pictures} in the decoder exceed the memory budget",
+53 -10
View File
@@ -139,8 +139,9 @@ impl Dpb {
let mut unused = (prev_ref_frame_num + 1) % max_frame_num;
// Only the last `max_num_ref_frames` "non-existing" frames can survive the sliding window, and
// inserting more than that evicts every earlier short-term reference. Skip straight to that tail
// (keeping the POC state walk for every missing frame_num) so a hostile gap of up to 65535
// frames costs at most `max_num_ref_frames` frame allocations instead of one per missing frame.
// (advancing the POC state across the skipped run in constant time) so a hostile gap of up to
// 65535 frames costs at most `max_num_ref_frames` frame allocations and POC updates instead of
// one per missing frame.
let gap = frame_num.wrapping_add(max_frame_num).wrapping_sub(unused) % max_frame_num;
let keep = u32::try_from(max_num_ref_frames.max(1)).unwrap_or(u32::MAX);
let skip = gap.saturating_sub(keep);
@@ -149,10 +150,17 @@ impl Dpb {
e.mark = RefMark::Unused;
}
self.prune();
for _ in 0..skip {
poc_state(unused);
unused = (unused + 1) % max_frame_num;
// The POC state only advances FrameNumOffset where frame_num decreases, so walking the
// skipped run is the same as visiting its first value, the wrap (if the run crosses
// it) and its last value: constant work however long the gap.
let last = (unused + (skip - 1) % max_frame_num) % max_frame_num;
poc_state(unused);
if last < unused {
poc_state(max_frame_num - 1);
poc_state(0);
}
poc_state(last);
unused = (last + 1) % max_frame_num;
}
let mut guard = 0;
while unused != frame_num && guard < max_frame_num {
@@ -438,14 +446,15 @@ impl Dpb {
}
}
} else {
while self.entries.len() >= self.capacity {
// Evict until there is room (frame_num gap concealment can leave the DPB over-full), so
// the store never holds more pictures than the memory budget was checked for.
while self.entries.len() >= self.capacity.max(1) {
if !self.bump(out) {
if let Some(pos) = self.entries.iter().position(|e| !e.needed_for_output) {
self.entries.remove(pos);
} else {
self.entries.remove(0);
}
break;
}
}
}
@@ -546,6 +555,37 @@ mod tests {
(made, walk, left)
}
/// (FrameNumOffset, prev frame_num) after `walk`, by the rule of `PocState::update_gap_frame`.
fn poc_after(walk: &[u32], prev: u32, max: u32) -> (u64, u32) {
walk.iter().fold((0, prev), |(off, prev), &f| (if prev > f { off + u64::from(max) } else { off }, f))
}
#[test]
fn gap_walk_matches_every_frame_num_for_any_skip() {
for (prev, cur) in [(1, 9), (10, 3), (14, 0), (15, 14), (0, 15), (7, 6)] {
for refs in 1..4 {
let mut dpb = Dpb::new();
let (_, walk, _) = fill(&mut dpb, prev, cur, 16, refs);
let every: Vec<u32> = (1..16).map(|i| (prev + i) % 16).take_while(|&f| f != cur).collect();
assert_eq!(poc_after(&walk, prev, 16), poc_after(&every, prev, 16), "{prev} -> {cur}, {refs} refs");
assert_eq!(walk.last(), every.last(), "{prev} -> {cur}, {refs} refs");
}
}
}
#[test]
fn reference_store_evicts_down_to_capacity() {
let mut dpb = Dpb::new();
dpb.capacity = 2;
// Gap concealment can leave more non-output references than the capacity.
for i in 0..5 {
short(&mut dpb, i, i);
}
let mut out = Vec::new();
assert!(!dpb.bump_for_space(RefMark::Short, 0, &mut out));
assert!(dpb.entries.len() < 2, "{} pictures left before storing into a 2-picture DPB", dpb.entries.len());
}
#[test]
fn huge_gap_makes_only_the_frames_that_stay_referenced() {
let mut dpb = Dpb::new();
@@ -553,8 +593,9 @@ mod tests {
short(&mut dpb, 2, 1);
let (made, walk, left) = fill(&mut dpb, 1, 65535, 65536, 4);
assert_eq!(made, 4);
// The POC state still walks every missing frame_num, in order.
assert_eq!(walk.len(), 65533);
// The POC state ends where walking every missing frame_num would leave it, in a few steps.
assert_eq!(poc_after(&walk, 1, 65536), poc_after(&(2..65535).collect::<Vec<_>>(), 1, 65536));
assert!(walk.len() <= 8, "{} POC updates", walk.len());
assert_eq!(walk.first(), Some(&2));
assert_eq!(walk.last(), Some(&65534));
assert_eq!(left, vec![(65531, true), (65532, true), (65533, true), (65534, true)]);
@@ -566,7 +607,9 @@ mod tests {
short(&mut dpb, 1, 65000);
let (made, walk, left) = fill(&mut dpb, 65000, 3, 65536, 2);
assert_eq!(made, 2);
assert_eq!(walk.len(), 538);
let every: Vec<u32> = (65001..65536).chain(0..3).collect();
assert_eq!(poc_after(&walk, 65000, 65536), poc_after(&every, 65000, 65536));
assert!(walk.len() <= 6, "{} POC updates", walk.len());
assert_eq!(walk.first(), Some(&65001));
assert_eq!(walk.last(), Some(&2));
assert_eq!(left, vec![(1, true), (2, true)]);
+8 -4
View File
@@ -346,8 +346,11 @@ impl<'a> SliceDecoder<'a> {
// Neighbour helpers
// ------------------------------------------------------------------------------------------
/// Set up the current macroblock address and neighbour availability.
pub fn start_mb(&mut self, addr: usize) {
/// Set up the current macroblock address and neighbour availability. A macroblock an earlier
/// slice of this picture already decoded is refused: overlapping slices would otherwise redo
/// the picture's reconstruction once per slice.
pub fn start_mb(&mut self, addr: usize) -> Result<()> {
ensure!(self.pic.mbs[addr].slice_num == u32::MAX, "macroblock {addr} overlaps an earlier slice");
self.mb_addr = addr;
self.mb_x = addr % self.mb_w;
self.mb_y = addr / self.mb_w;
@@ -364,6 +367,7 @@ impl<'a> SliceDecoder<'a> {
];
self.mv_done = 0;
self.pic.mbs[addr] = MbState { slice_num: sn, ..MbState::default() };
Ok(())
}
#[inline]
@@ -1467,7 +1471,7 @@ impl<'a> SliceDecoder<'a> {
let run = r.read_ue()? as usize;
ensure!(addr + run <= total, "mb_skip_run out of range");
for _ in 0..run {
self.start_mb(addr);
self.start_mb(addr)?;
self.decode_skip()?;
self.pic.mb_done(addr);
addr += 1;
@@ -1477,7 +1481,7 @@ impl<'a> SliceDecoder<'a> {
}
ensure!(addr < total, "macroblock address {addr} out of range");
}
self.start_mb(addr);
self.start_mb(addr)?;
self.decode_mb_cavlc(r)?;
self.pic.mb_done(addr);
addr += 1;
+56 -1
View File
@@ -113,6 +113,7 @@ fn avcc_rejects_a_valid_picture_slice_without_starting_a_picture() {
poc_lsb: Some(0),
poc_delta: None,
wt: None,
first_mb: 0,
},
false,
);
@@ -153,6 +154,8 @@ struct Pic {
/// Explicit weights (luma, cb, cr) as (weight, offset) for refIdx 0 of list 0 and list 1;
/// log2 denominators are 5 (luma) and 3 (chroma).
wt: Option<[[(i32, i32); 3]; 2]>,
/// first_mb_in_slice (`mbs` then lists the slice's macroblocks from there).
first_mb: u32,
}
impl Pic {
@@ -165,7 +168,7 @@ impl Pic {
}
fn slice_header(w: &mut BitWriter, p: &Pic, cabac: bool) {
w.write_ue(0); // first_mb
w.write_ue(p.first_mb);
w.write_ue(if p.b {
6
} else if p.is_p() {
@@ -660,3 +663,55 @@ fn frame_num_gap_is_filled_when_the_sps_allows_gaps() {
assert_eq!(out.len(), 2);
assert_eq!(dec.stats().frame_num_gaps, 1);
}
/// An IDR picture followed by `pictures` reference P pictures of P_Skip macroblocks.
fn skip_stream(pictures: u32) -> (Vec<u8>, Vec<Vec<u8>>) {
let mut seed = 11u32;
let a: Vec<Option<Vec<u8>>> = (0..NMB).map(|_| Some(random_mb(&mut seed))).collect();
let mut head = sps(66, 2, false, PocType::T2);
head.extend(pps(false, false));
head.extend(slice_nal(&Pic { idr: true, frame_num: 0, ref_idc: 3, mbs: a, ..Default::default() }, false));
let ps = (1..=pictures).map(|k| slice_nal(&Pic { frame_num: k % 16, ref_idc: 2, mbs: vec![None; NMB], ..Default::default() }, false)).collect();
(head, ps)
}
#[test]
fn overlapping_slices_are_not_reconstructed_twice() {
let (head, ps) = skip_stream(1);
let mut once = Decoder::with_threads(1);
let mut out = once.decode(&[head.clone(), ps[0].clone()].concat(), 0).unwrap();
out.extend(once.flush());
assert!(once.take_error().is_none());
assert_eq!(out.len(), 2);
// A second slice of the same picture starting at macroblock 1 covers those macroblocks again.
let again = slice_nal(&Pic { frame_num: 1, ref_idc: 2, mbs: vec![None; NMB - 1], first_mb: 1, ..Default::default() }, false);
let mut twice = Decoder::with_threads(1);
let result = twice.decode(&[head, ps[0].clone(), again].concat(), 0);
twice.flush();
let err = result.err().or_else(|| twice.take_error()).expect("overlapping slice refused");
assert!(err.to_string().contains("overlaps"), "{err}");
}
#[test]
fn output_queue_counts_against_the_memory_budget() {
let (head, ps) = skip_stream(40);
let sps_nal = sps(66, 2, false, PocType::T2);
let parsed = crate::params::Sps::parse(&deckcraft_bitstream::unescape_rbsp(&sps_nal[5..])).unwrap();
let one = (NMB * 256 + 2 * NMB * 64) as u64;
let budget = (parsed.max_dpb_frames() + 1 + 3) as u64 * one;
// One access unit per call: output drains as it goes, and the stream fits.
let mut dec = Decoder::with_threads(1);
dec.set_memory_budget(budget);
let mut out = dec.decode(&head, 0).unwrap();
for (i, p) in ps.iter().enumerate() {
out.extend(dec.decode(p, i as i64 + 1).unwrap());
}
out.extend(dec.flush());
assert!(dec.take_error().is_none());
assert_eq!(out.len(), 41);
// Every access unit in one call: the pictures held for output stop fitting the budget.
let mut dec = Decoder::with_threads(1);
dec.set_memory_budget(budget);
let err = dec.decode(&[head, ps.concat()].concat(), 0).expect_err("output queue is over budget");
assert!(err.to_string().contains("memory budget"), "{err}");
}