diff --git a/crates/gpu/src/ctx.rs b/crates/gpu/src/ctx.rs index 33e02fbe4..7bbbbc68d 100644 --- a/crates/gpu/src/ctx.rs +++ b/crates/gpu/src/ctx.rs @@ -499,11 +499,31 @@ impl Gpu { self.over_limit(bytes.len() as u64); return Buf { buf: Some(Tracked::new(self.placeholder())), len }; } - let buf = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: None, - contents: bytes, - usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST, - }); + let usage = wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST; + if let Some(size) = wgpu::BufferSize::new(bytes.len() as u64).filter(|s| s.get() % wgpu::COPY_BUFFER_ALIGNMENT == 0) { + // through the queue: the bytes are copied once, into a staging buffer the next submission copies from, in + // parallel parts (`create_buffer_init` zeroes a staging buffer first and copies over that, two passes on + // one thread over a full-size photo's ~290 MB) + let buf = self.device.create_buffer(&wgpu::BufferDescriptor { label: None, size: size.get(), usage, mapped_at_creation: false }); + if let Some(mut view) = self.queue.write_buffer_with(&buf, 0, size) { + const PART: usize = 1 << 20; + { + let (whole, mut tail) = view.slice(..).into_chunks::(); + let (src, src_tail) = bytes.as_chunks::(); + let mut parts: Vec<_> = whole.into_iter().zip(src).map(Some).collect(); + lightcraft_raster::par_rows(&mut parts, 1, |_, part| { + for (d, s) in part.iter_mut().filter_map(Option::take) { + wgpu::WriteOnly::<[u8]>::from(d).copy_from_slice(s); + } + }); + tail.copy_from_slice(src_tail); + } + // (dropping the view schedules the copy) + drop(view); + return Buf { buf: Some(Tracked::new(buf)), len }; + } + } + let buf = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: None, contents: bytes, usage }); Buf { buf: Some(Tracked::new(buf)), len } } @@ -612,7 +632,17 @@ impl Gpu { Ok(d) => d, Err(e) => return failed(format!("{e:?}")), }; - let out: Vec = bytemuck::cast_slice(&data).to_vec(); + // copied out in parallel parts: one thread reads mapped memory slowly (~2 GB/s, so ~45 ms for a 24 MP + // export's pixels), and the new vector's pages are first touched there too + let src: &[u8] = &data; + // (`len` counts 32-bit words: as many `T` as they fill) + let mut out = vec![::zeroed(); src.len() / std::mem::size_of::().max(1)]; + const PART: usize = 1 << 20; + lightcraft_raster::par_rows(bytemuck::cast_slice_mut::(&mut out), PART, |k, part| { + if let Some(s) = src.get(k * PART..k * PART + part.len()) { + part.copy_from_slice(s); + } + }); drop(data); staging.unmap(); out @@ -811,6 +841,26 @@ mod tests { Some(stdout) } + /// An upload reaches the device whole: it is copied into staging memory in parallel parts, the last one + /// shorter than the others (and a small upload is that last part alone). + #[test] + fn uploads_read_back_exactly() { + let Some(g) = crate::device() else { + lightcraft_testkit::skip!(Gpu, "no GPU device"); + return; + }; + for len in [1usize, 1000, (3 << 20) / 4 + 777] { + let data: Vec = (0..len as u32).map(|i| i.wrapping_mul(2_654_435_761)).collect(); + let b = g.upload(&data); + let back: Vec = g.finish_and_read(g.encoder(), &b, len); + assert!(back == data, "{len} values"); + } + // read back as wider values (`len` counts 32-bit words), as `read_rgb` does + let data: Vec = (0..3 << 20).collect(); + let b = g.upload(&data); + let back: Vec<[u32; 3]> = g.finish_and_read(g.encoder(), &b, data.len()); + assert!(back.len() == 1 << 20 && back.as_flattened() == data.as_slice()); + } /// Issue #1858: once a Vulkan or DX12 device is lost, wgpu 30 panics in `Device::poll` instead of /// returning an error. A render's readback panicked; while it unwound, a buffer's way back to /// the pool polls again from a destructor, and that second panic aborted the app. diff --git a/crates/raw/src/demosaic.rs b/crates/raw/src/demosaic.rs index b8a3c599a..75f7674ac 100644 --- a/crates/raw/src/demosaic.rs +++ b/crates/raw/src/demosaic.rs @@ -15,7 +15,7 @@ mod bilinear; mod ppg; mod xtrans; -use crate::{Cfa, Normalized, Rgb32f}; +use crate::{Cfa, Normalized, Rect, Rgb32f}; use serde::{Deserialize, Serialize}; #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -114,6 +114,20 @@ pub fn demosaic(n: &Normalized, method: Method) -> Rgb32f { } } +/// [`demosaic`] with [`Method::Ahd`] of the window `r` alone, for Bayer mosaics (`None` for any other data, or a +/// window not inside the mosaic): the same pixels as demosaicing it all and cropping that to `r`, without +/// demosaicing the rest or copying the window out of it. +pub(crate) fn ahd_window(n: &Normalized, r: Rect) -> Option { + let (w, h) = (n.width, n.height); + let cfa = n.cfa.as_ref()?; + let inside = r.x.checked_add(r.width).is_some_and(|e| e <= w) && r.y.checked_add(r.height).is_some_and(|e| e <= h); + if n.cpp != 1 || !cfa.is_bayer() || w < 4 || h < 4 || n.data.len() != w * h || r.width == 0 || r.height == 0 || !inside { + return None; + } + let m = Mosaic { w, h, data: &n.data, cfa }; + Some(ahd::ahd_window(&m, r.x, r.y, r.width, r.height)) +} + /// Build a mosaic from an RGB image by sampling each pixel's CFA colour (for tests and synthetic data). pub fn mosaic_from_rgb(img: &Rgb32f, cfa: &Cfa) -> Normalized { let data = (0..img.height).flat_map(|y| (0..img.width).map(move |x| (x, y))).map(|(x, y)| img.get(x, y)[cfa.color_at(x, y) as usize]).collect(); @@ -192,6 +206,26 @@ mod tests { assert!(out.data.iter().all(|p| (p[0] - 0.3).abs() < 1e-5 && (p[1] - 0.55).abs() < 1e-5 && (p[2] - 0.8).abs() < 1e-5)); } + /// A window demosaiced alone is the same, bit for bit, as that part of the whole picture: at the edges and + /// corners (where the mosaic is reflected), in the middle, across tile seams, at every CFA phase. + #[test] + fn ahd_window_is_the_crop_of_the_whole() { + let img = edgy_scene(300, 270); + for pat in ["RGGB", "GBRG"] { + let n = mosaic_from_rgb(&img, &Cfa::bayer(pat).unwrap()); + let whole = demosaic(&n, Method::Ahd); + for r in + [Rect::new(0, 0, 300, 270), Rect::new(0, 0, 7, 5), Rect::new(1, 3, 298, 266), Rect::new(121, 130, 140, 9), Rect::new(293, 263, 7, 7)] + { + let part = ahd_window(&n, r).unwrap(); + let want = whole.clone().into_crop(r.x, r.y, r.width, r.height); + assert_eq!((part.width, part.height), (r.width, r.height)); + assert!(part.data.iter().zip(&want.data).all(|(a, b)| a.map(f32::to_bits) == b.map(f32::to_bits)), "{pat} {r:?}"); + } + assert!(ahd_window(&n, Rect::new(1, 0, 300, 270)).is_none(), "a window past the mosaic is refused"); + } + } + #[test] fn known_samples_are_preserved() { let img = edgy_scene(40, 32); diff --git a/crates/raw/src/demosaic/ahd.rs b/crates/raw/src/demosaic/ahd.rs index d9fc27d76..6afcd9251 100644 --- a/crates/raw/src/demosaic/ahd.rs +++ b/crates/raw/src/demosaic/ahd.rs @@ -21,19 +21,24 @@ const TILE: usize = 128; const MARGIN: usize = 6; pub(crate) fn ahd(m: &Mosaic) -> Rgb32f { - let (w, h) = (m.w, m.h); + ahd_window(m, 0, 0, m.w, m.h) +} + +/// The `w × h` pixels at (`x`, `y`) of [`ahd`]'s picture (inside the mosaic), demosaiced alone: every pixel comes +/// from the mosaic around it, so they are the same as when the whole picture is demosaiced and then cropped. +pub(crate) fn ahd_window(m: &Mosaic, x: usize, y: usize, w: usize, h: usize) -> Rgb32f { let mut out = Rgb32f::new(w, h); if w == 0 || h == 0 { return out; } // one row of tiles per task, its tiles in parallel too, written straight into the picture out.data.par_chunks_mut((w * TILE).max(1)).enumerate().for_each(|(ty, band)| { - let (y0, th) = (ty * TILE, band.len() / w.max(1)); + let (y0, th) = (y + ty * TILE, band.len() / w.max(1)); let tiles: Vec<(usize, usize, Vec<[f32; 3]>)> = (0..w.div_ceil(TILE)) .into_par_iter() .map(|tx| { let (x0, tw) = (tx * TILE, TILE.min(w - tx * TILE)); - (x0, tw, tile(m, x0, y0, tw, th)) + (x0, tw, tile(m, x + x0, y0, tw, th)) }) .collect(); for (x0, tw, px) in tiles { diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rs index 6626f10cb..17152e80f 100644 --- a/crates/raw/src/lib.rs +++ b/crates/raw/src/lib.rs @@ -866,6 +866,15 @@ impl RawImage { /// The result is camera RGB (not white balanced), white level = 1.0, not oriented. pub fn develop(&self, method: Method) -> Result { let n = self.normalized()?; + // with no `OpcodeList3` to apply to the whole picture first, AHD demosaics just the default crop (the same + // pixels, without demosaicing the margin and then copying the crop out of the full picture) + if self.opcodes.list3.is_empty() && method == Method::Ahd { + let c = self.develop_crop(n.width, n.height); + if let Some(rgb) = demosaic::ahd_window(&n, c) { + drop(n); + return Ok(self.stretched(rgb)); + } + } let rgb = demosaic(&n, method); drop(n); Ok(self.finish_demosaiced(rgb)) @@ -878,6 +887,11 @@ impl RawImage { let c = self.develop_crop(rgb.width, rgb.height); let rgb = if c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height { rgb } else { rgb.into_crop(c.x, c.y, c.width, c.height) }; + self.stretched(rgb) + } + + /// The cropped camera RGB with its rows stretched for non-square pixels. + fn stretched(&self, rgb: Rgb32f) -> Rgb32f { let h = stretched_height(rgb.height, self.pixel_aspect); if h == rgb.height { return rgb;