//! `lightcraft-heif`: the optional HEIF / HEIC (iPhone and Mac photo) decoder. //! //! `heic-rs`, a pure-Rust HEIF container and HEVC still-picture decoder, reads the container and //! reconstructs the coded pictures (bit-exact against libde265 on iPhone photos); this crate turns //! them into RGB itself ([`ycc`]) so the colours match libheif: it honours the HEVC VUI's range //! and matrix ([`vui`]) where heic-rs assumes BT.709 limited range, and upsamples chroma like //! libheif. Single pictures and grid-tiled photos, 8- to 16-bit, monochrome, alpha auxiliary //! images, ICC, EXIF and XMP. The API is plain data ([`Info`], [`Decoded`], [`Error`]) so the //! crate knows nothing about the rest of LightCraft; `lightcraft-codecs` adapts it behind its //! `heif` feature. Shares heic-rs and its regression fixtures with PhotoCraft's `photocraft-heif`. //! //! HEIF records orientation in the container (`irot`/`imir`, plus a `clap` crop), not in EXIF. //! [`Options::apply_transforms`] applies them; with it off, the pixels come back as coded. //! Auxiliary images other than alpha (HDR gain maps, depth maps) are ignored. //! //! Never panics: heic-rs is young, so every call into it runs under `catch_unwind` and a panic //! inside it becomes [`Error::Malformed`]. #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] mod orient; /// Lossless synthetic HEIF files for tests, here and in the crates above (feature `testdata`). #[cfg(any(test, feature = "testdata"))] #[doc(hidden)] pub mod testdata; mod vui; mod ycc; use std::borrow::Cow; use heic_rs::context::Context; use heic_rs::hevc::Frame; use heic_rs::props::ItemProps; use heic_rs::props::colr::Range; /// What a HEIF file declares, read from the container alone (no pixel is decoded). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Info { /// Size after the container transforms (rotation, mirror, crop): what a viewer shows. pub width: u32, pub height: u32, /// Size as coded, before the transforms. pub coded_width: u32, pub coded_height: u32, /// The primary image has an alpha auxiliary image. pub has_alpha: bool, /// Bits per sample as coded (8 for most photos, 10 for HDR-capable ones). pub bit_depth: u8, } /// Decode settings. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Options { /// Refuse images with more pixels than this (checked before decoding). pub max_pixels: u64, /// Apply the container's rotation, mirror and crop (`irot`/`imir`/`clap`). pub apply_transforms: bool, } impl Default for Options { fn default() -> Self { Options { max_pixels: 1 << 28, apply_transforms: true } } } /// How the file describes its colour: the code points of ITU-T H.273, from the `colr` `nclx` /// box, else the HEVC VUI, else the standard defaults (2 = unspecified, limited range). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Colour { pub primaries: u16, pub transfer: u16, pub matrix: u16, pub full_range: bool, } /// A decoded image: interleaved RGB or RGBA, 16 bits per sample whatever the coded depth, /// row-major, no padding. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Decoded { pub width: u32, pub height: u32, /// Four channels (RGBA) instead of three (RGB). pub has_alpha: bool, /// Bits per sample as coded. pub bit_depth: u8, pub data: Vec, pub colour: Colour, /// The primary image's ICC profile. pub icc: Option>, /// EXIF as a TIFF structure (without HEIF's offset header). Its Orientation tag only mirrors /// what the container says; it is returned verbatim. pub exif: Option>, /// The XMP packet. pub xmp: Option, } /// Why a file could not be decoded. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Error { /// A valid file using something this decoder does not handle (image sequences, overlays…). Unsupported(&'static str), /// The image is larger than [`Options::max_pixels`] or holds an oversized box. Limit(String), /// Broken or truncated data (or a decoder bug, reported the same way). Malformed(String), } impl std::fmt::Display for Error { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Error::Unsupported(m) => write!(f, "unsupported HEIF: {m}"), Error::Limit(m) => write!(f, "limit exceeded: {m}"), Error::Malformed(m) => write!(f, "malformed HEIF: {m}"), } } } impl std::error::Error for Error {} const SEQUENCE: &str = "this file holds an image sequence; only HEIF still images can be opened"; /// heic-rs 0.1.1 reconstructs 4:2:0 and monochrome pictures exactly but fails on the 4:2:2 and /// 4:4:4 ones some cameras write (Canon and Sony "HIF"); those failures are reported as this. const CHROMA_422_444: &str = "4:2:2 and 4:4:4 HEIF images (as some Canon and Sony cameras write) can't be decoded yet"; pub(crate) fn err(e: heic_rs::Error) -> Error { match e { heic_rs::Error::Unsupported(what) => Error::Unsupported(what), // The primary item is neither an HEVC picture nor a grid of them. heic_rs::Error::MissingBox("hvcC") => Error::Unsupported("the image is an overlay, an identity derivation or not HEVC-coded"), heic_rs::Error::PixelLimit { .. } | heic_rs::Error::BoxTooLarge { .. } => Error::Limit(e.to_string()), e => Error::Malformed(e.to_string()), } } /// Runs `f`, turning a heic-rs panic into an error. Fuzzing in the sibling app found two /// out-of-range slices in heic-rs 0.1.1 on malformed files: in its box parser (`boxes.rs:130`) /// and during reconstruction (`hevc/decode/recon.rs:70`). fn guarded(f: impl FnOnce() -> Result) -> Result { std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).unwrap_or_else(|_| Err(Error::Malformed("the HEIF decoder failed on this file".into()))) } /// Reads the container's declared size, depth and alpha without decoding pixels, so callers can /// check their limits first. Also refuses a file cut short: every coded picture of the primary /// image must lie within `bytes`. pub fn probe(bytes: &[u8]) -> Result { guarded(|| probe_unguarded(bytes)) } fn probe_unguarded(bytes: &[u8]) -> Result { let info = heic_rs::probe(bytes).map_err(|e| match e { // No `meta` box: no still image, only a `moov` image sequence (a video track). heic_rs::Error::MissingBox("meta") => Error::Unsupported(SEQUENCE), e => err(e), })?; let ctx = Context::open(bytes).map_err(err)?; let id = ctx.meta.primary; let coded = match ctx.grid(id).map_err(err)? { Some((_, tiles)) => tiles, None => vec![id], }; for item in coded { // Borrowed from `bytes` unless split across extents: cheap. ctx.item_data(item).map_err(err)?; } Ok(Info { width: info.width, height: info.height, coded_width: info.coded_width, coded_height: info.coded_height, has_alpha: info.has_alpha, bit_depth: info.bit_depth, }) } /// Decodes the primary image, with its metadata. pub fn decode(bytes: &[u8], options: &Options) -> Result { guarded(|| decode_unguarded(bytes, options)) } /// Decodes the smallest thumbnail image the file carries whose long edge is at least /// `min_edge`, if any (iPhones store one of about 320 Ă— 240). Transforms are applied as for the /// primary image; the metadata (and, when the thumbnail has no profile of its own, the ICC /// profile) are the primary image's. pub fn decode_thumbnail(bytes: &[u8], min_edge: u32, options: &Options) -> Result, Error> { guarded(|| { let ctx = Context::open(bytes).map_err(err)?; let primary = ctx.meta.primary; let mut best: Option<(u32, u32)> = None; for r in ctx.meta.refs.iter().filter(|r| r.kind == heic_rs::meta::iref::RefKind::Thmb && r.to.contains(&primary)) { let Ok(p) = ctx.props(r.from) else { continue }; let Ok((w, h)) = ctx.coded_size(r.from, &p) else { continue }; let edge = w.max(h); if edge >= min_edge && best.is_none_or(|(_, e)| edge < e) { best = Some((r.from, edge)); } } let Some((id, _)) = best else { return Ok(None) }; let mut decoded = decode_item(&ctx, id, options, false)?; if decoded.icc.is_none() { decoded.icc = item_icc(&ctx, primary); } (decoded.exif, decoded.xmp) = metadata(&ctx); Ok(Some(decoded)) }) } /// An item's ICC profile; a grid whose own properties carry none takes its first tile's (as on /// some iPhones). fn item_icc(ctx: &Context<'_>, id: u32) -> Option> { let p = ctx.props(id).ok()?; let icc = p.icc.or_else(|| { let (_, tiles) = ctx.grid(id).ok()??; ctx.props(*tiles.first()?).ok()?.icc }); icc.map(<[u8]>::to_vec) } fn decode_unguarded(bytes: &[u8], options: &Options) -> Result { // Refuses sequences with a clear message before anything else is read. probe_unguarded(bytes)?; let ctx = Context::open(bytes).map_err(err)?; let mut decoded = decode_item(&ctx, ctx.meta.primary, options, true)?; let (exif, xmp) = metadata(&ctx); decoded.exif = exif; decoded.xmp = xmp; Ok(decoded) } /// Decodes one image item (a coded picture or a grid of them) to RGB(A), alpha included when /// `with_alpha` and the item has an alpha auxiliary image. fn decode_item(ctx: &Context<'_>, id: u32, options: &Options, with_alpha: bool) -> Result { let p = ctx.props(id).map_err(err)?; let limit = options.max_pixels; // The declared sizes are refused before anything is decoded or allocated. let (cw, ch) = ctx.coded_size(id, &p).map_err(err)?; heic_rs::image::check_pixels(cw, ch, limit).map_err(err)?; let (tw, th) = heic_rs::transform::transformed_size(cw, ch, &p.transforms).map_err(err)?; heic_rs::image::check_pixels(tw, th, limit).map_err(err)?; let coded = Coded::decode(ctx, id, &p, limit)?; let planes = coded.planes()?; let colour = colour(&p, &coded.first_props); let signal = ycc::Signal { matrix: colour.matrix, full_range: colour.full_range }; let alpha_id = if with_alpha { ctx.alpha_item(id).map_err(err)? } else { None }; let (w, h) = planes.size(); let mode = if options.apply_transforms && orient::odd_crop(w, h, &p.transforms)? { ycc::Upsampling::Bilinear } else { ycc::Upsampling::Nearest }; let mut data = ycc::to_rgb16(&planes, signal, alpha_id.is_some(), mode) .ok_or_else(|| Error::Malformed("the decoded pictures don't cover the image".into()))?; if let Some(aux) = alpha_id { let ap = ctx.props(aux).map_err(err)?; let alpha = Coded::decode(ctx, aux, &ap, limit)?; let alpha_planes = alpha.planes()?; if alpha_planes.size() != (w, h) { return Err(Error::Unsupported("an alpha image of a different size than the picture")); } ycc::put_alpha(&alpha_planes, &mut data).ok_or_else(|| Error::Malformed("the alpha image is incomplete".into()))?; } let bit_depth = planes.bit_depth(); let has_alpha = alpha_id.is_some(); let mut img = orient::Pixels { width: w, height: h, ch: if has_alpha { 4 } else { 3 }, data }; if options.apply_transforms { img = orient::apply(img, &p.transforms)?; } Ok(Decoded { width: u32::try_from(img.width).map_err(|_| Error::Limit("image too wide".into()))?, height: u32::try_from(img.height).map_err(|_| Error::Limit("image too tall".into()))?, has_alpha, bit_depth, data: img.data, colour, // A grid whose own properties carry no profile takes its tiles' (as on some iPhones). icc: p.icc.or(coded.first_props.icc).map(<[u8]>::to_vec), exif: None, xmp: None, }) } /// The decoded pictures of an item: one, or a grid's tiles in `dimg` order. struct Coded<'a> { frames: Vec, grid: Option, /// The properties of the first coded picture (the item itself, or the first tile): where the /// HEVC configuration and its VUI live. first_props: ItemProps<'a>, /// A failed 4:2:2 / 4:4:4 picture, for the error message. chroma_format: u8, } impl<'a> Coded<'a> { fn decode(ctx: &Context<'a>, id: u32, p: &ItemProps<'a>, limit: u64) -> Result, Error> { let (grid, ids) = match ctx.grid(id).map_err(err)? { Some((g, tiles)) => (Some(g), tiles), None => (None, vec![id]), }; let first = *ids.first().ok_or_else(|| Error::Malformed("grid lists no tiles".into()))?; let first_props = if first == id { p.clone() } else { ctx.props(first).map_err(err)? }; let chroma_format = first_props.hvcc.as_ref().map_or(1, |c| c.chroma_format); let decode_one = |t: &u32| -> Result { let tp = ctx.props(*t).map_err(err)?; let hvcc = tp.hvcc.as_ref().ok_or(heic_rs::Error::MissingBox("hvcC")).map_err(err)?; let data = ctx.item_data(*t).map_err(err)?; let nals = hvcc.split_nals(&data).map_err(err)?; let params = hvcc.parameter_sets(); if params.is_empty() { return Err(Error::Malformed("hvcC carries no parameter sets".into())); } let frame = heic_rs::hevc::decode_still(¶ms, &nals).map_err(err)?; frame.validate().map_err(err)?; heic_rs::image::check_pixels(frame.width, frame.height, limit).map_err(err)?; Ok(frame) }; #[cfg(not(target_arch = "wasm32"))] let frames: Result, Error> = { use rayon::prelude::*; ids.par_iter().map(decode_one).collect() }; #[cfg(target_arch = "wasm32")] let frames: Result, Error> = ids.iter().map(decode_one).collect(); let frames = frames.map_err(|e| match e { Error::Malformed(_) if chroma_format >= 2 => Error::Unsupported(CHROMA_422_444), e => e, })?; Ok(Coded { frames, grid, first_props, chroma_format }) } fn planes(&self) -> Result, Error> { match (&self.grid, self.frames.as_slice()) { (Some(g), tiles) => heic_rs::grid::Mosaic::new(g, tiles).map(ycc::Planes::Grid).map_err(err), (None, [frame]) => Ok(ycc::Planes::Picture(frame)), (None, _) => Err(Error::Malformed(format!("expected one coded picture (chroma format {})", self.chroma_format))), } } } /// The colour description in force: `nclx` on the item (or, for a grid without one, on its /// first tile), else the VUI of its (first) coded picture, else the H.265 defaults. fn colour(p: &ItemProps<'_>, coded: &ItemProps<'_>) -> Colour { if let Some(n) = p.nclx.or(coded.nclx) { return Colour { primaries: n.primaries, transfer: n.transfer, matrix: n.matrix_code, full_range: n.range == Range::Full }; } let s = coded.hvcc.as_ref().and_then(|c| vui::signal(&c.parameter_sets())).unwrap_or_default(); Colour { primaries: s.primaries.into(), transfer: s.transfer.into(), matrix: s.matrix.into(), full_range: s.full_range } } /// The primary image's EXIF (TIFF structure) and XMP packet. Metadata is a courtesy: a broken /// metadata item never fails a decode whose pixels came out fine. fn metadata(ctx: &Context<'_>) -> (Option>, Option) { let id = ctx.meta.primary; let exif = ctx.exif(id).ok().flatten().map(<[u8]>::to_vec); let xmp = ctx.xmp_item(id).and_then(|x| ctx.item_data(x).ok()).and_then(|d| match d { Cow::Borrowed(b) => String::from_utf8(b.to_vec()).ok(), Cow::Owned(b) => String::from_utf8(b).ok(), }); (exif, xmp.map(|x| x.trim_end_matches('\0').to_string())) } #[cfg(test)] mod tests_synthetic; #[cfg(test)] mod tests { use super::*; /// Found by PhotoCraft's `decode_heif` fuzz target: a malformed box makes heic-rs 0.1.1 slice /// out of range (`boxes.rs:130`). It must be an error, not a crash. const HEIC_RS_BOX_PANIC: [u8; 72] = [ 0x00, 0x00, 0x00, 0x24, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x69, 0x66, 0x31, 0x4d, 0x69, 0x50, 0x72, 0x6d, 0x69, 0x61, 0x66, 0x4d, 0x69, 0x48, 0x42, 0x72, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x65, 0x74, 0x61, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x75, 0x75, 0x69, 0x64, 0x00, 0x00, 0x00, 0x00, 0x03, 0x08, 0x08, 0x08, 0x00, 0x03, 0x01, 0x03, 0x70, 0x00, 0xa8, 0x00, ]; #[test] fn a_heic_rs_panic_is_a_malformed_error() { for apply_transforms in [true, false] { let r = decode(&HEIC_RS_BOX_PANIC, &Options { apply_transforms, ..Default::default() }); assert!(matches!(r, Err(Error::Malformed(_))), "{r:?}"); } } #[test] fn image_sequence_is_unsupported_with_a_clear_message() { // An ftyp and nothing else: a HEIF image sequence would have a moov here instead of a meta. let bytes = b"\0\0\0\x18ftypmsf1\0\0\0\0msf1hevc"; for r in [probe(bytes).map(|_| ()), decode(bytes, &Options::default()).map(|_| ())] { assert!(matches!(&r, Err(Error::Unsupported(m)) if m.contains("sequence")), "{r:?}"); } } #[test] fn garbage_is_an_error() { for bytes in [&b""[..], b"\0\0\0\x18ftypheic\0\0\0\0mif1heic", &[0xFF; 64]] { assert!(probe(bytes).is_err()); assert!(decode(bytes, &Options::default()).is_err()); assert!(decode_thumbnail(bytes, 1, &Options::default()).is_err()); } } #[test] fn errors_display_their_kind() { assert!(Error::Unsupported("x").to_string().contains("unsupported")); assert!(Error::Limit("x".into()).to_string().contains("limit")); assert!(Error::Malformed("x".into()).to_string().contains("malformed")); } }