Cut serial work from raw development and GPU transfers

- RawImage::develop demosaics only the default crop with AHD when there
  is no OpcodeList3 (ahd_window), instead of demosaicing the margin too
  and then copying the crop out of the full picture on one thread.
- Gpu::upload writes through queue.write_buffer_with and copies into
  the staging buffer in parallel parts. create_buffer_init zeroed a
  staging buffer and then copied over it, two single-threaded passes
  over a full-size photo's ~290 MB.
- finish_and_read copies the mapped readback out in parallel parts (one
  thread read the 91 MB of a 24 MP export at ~2 GB/s, ~45 ms).

Output is byte-identical (ahd_window_is_the_crop_of_the_whole,
uploads_read_back_exactly).

Export of a 24 MP ARW to full-size JPEG (examples/export_bench, 75
interleaved runs each on a shared 16-thread Ryzen):
main 802 ms median / 614 ms min -> this branch 647 / 525 (-19.3% / -14.5%).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

crates/gpu/src/ctx.rs

@@ -499,11 +499,31 @@ impl Gpu {
             self.over_limit(bytes.len() as u64);
             return Buf { buf: Some(Tracked::new(self.placeholder())), len };
         }
-        let buf = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
-            label: None,
-            contents: bytes,
-            usage: wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST,
-        });
+        let usage = wgpu::BufferUsages::STORAGE | wgpu::BufferUsages::COPY_SRC | wgpu::BufferUsages::COPY_DST;
+        if let Some(size) = wgpu::BufferSize::new(bytes.len() as u64).filter(|s| s.get() % wgpu::COPY_BUFFER_ALIGNMENT == 0) {
+            // through the queue: the bytes are copied once, into a staging buffer the next submission copies from, in
+            // parallel parts (`create_buffer_init` zeroes a staging buffer first and copies over that, two passes on
+            // one thread over a full-size photo's ~290 MB)
+            let buf = self.device.create_buffer(&wgpu::BufferDescriptor { label: None, size: size.get(), usage, mapped_at_creation: false });
+            if let Some(mut view) = self.queue.write_buffer_with(&buf, 0, size) {
+                const PART: usize = 1 << 20;
+                {
+                    let (whole, mut tail) = view.slice(..).into_chunks::<PART>();
+                    let (src, src_tail) = bytes.as_chunks::<PART>();
+                    let mut parts: Vec<_> = whole.into_iter().zip(src).map(Some).collect();
+                    lightcraft_raster::par_rows(&mut parts, 1, |_, part| {
+                        for (d, s) in part.iter_mut().filter_map(Option::take) {
+                            wgpu::WriteOnly::<[u8]>::from(d).copy_from_slice(s);
+                        }
+                    });
+                    tail.copy_from_slice(src_tail);
+                }
+                // (dropping the view schedules the copy)
+                drop(view);
+                return Buf { buf: Some(Tracked::new(buf)), len };
+            }
+        }
+        let buf = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: None, contents: bytes, usage });
         Buf { buf: Some(Tracked::new(buf)), len }
     }
 
@@ -612,7 +632,17 @@ impl Gpu {
             Ok(d) => d,
             Err(e) => return failed(format!("{e:?}")),
         };
-        let out: Vec<T> = bytemuck::cast_slice(&data).to_vec();
+        // copied out in parallel parts: one thread reads mapped memory slowly (~2 GB/s, so ~45 ms for a 24 MP
+        // export's pixels), and the new vector's pages are first touched there too
+        let src: &[u8] = &data;
+        // (`len` counts 32-bit words: as many `T` as they fill)
+        let mut out = vec![<T as bytemuck::Zeroable>::zeroed(); src.len() / std::mem::size_of::<T>().max(1)];
+        const PART: usize = 1 << 20;
+        lightcraft_raster::par_rows(bytemuck::cast_slice_mut::<T, u8>(&mut out), PART, |k, part| {
+            if let Some(s) = src.get(k * PART..k * PART + part.len()) {
+                part.copy_from_slice(s);
+            }
+        });
         drop(data);
         staging.unmap();
         out
@@ -811,6 +841,26 @@ mod tests {
         Some(stdout)
     }
 
+    /// An upload reaches the device whole: it is copied into staging memory in parallel parts, the last one
+    /// shorter than the others (and a small upload is that last part alone).
+    #[test]
+    fn uploads_read_back_exactly() {
+        let Some(g) = crate::device() else {
+            lightcraft_testkit::skip!(Gpu, "no GPU device");
+            return;
+        };
+        for len in [1usize, 1000, (3 << 20) / 4 + 777] {
+            let data: Vec<u32> = (0..len as u32).map(|i| i.wrapping_mul(2_654_435_761)).collect();
+            let b = g.upload(&data);
+            let back: Vec<u32> = g.finish_and_read(g.encoder(), &b, len);
+            assert!(back == data, "{len} values");
+        }
+        // read back as wider values (`len` counts 32-bit words), as `read_rgb` does
+        let data: Vec<u32> = (0..3 << 20).collect();
+        let b = g.upload(&data);
+        let back: Vec<[u32; 3]> = g.finish_and_read(g.encoder(), &b, data.len());
+        assert!(back.len() == 1 << 20 && back.as_flattened() == data.as_slice());
+    }
     /// Issue #1858: once a Vulkan or DX12 device is lost, wgpu 30 panics in `Device::poll` instead of
     /// returning an error. A render's readback panicked; while it unwound, a buffer's way back to
     /// the pool polls again from a destructor, and that second panic aborted the app.

crates/raw/src/demosaic.rs

@@ -15,7 +15,7 @@ mod bilinear;
 mod ppg;
 mod xtrans;
 
-use crate::{Cfa, Normalized, Rgb32f};
+use crate::{Cfa, Normalized, Rect, Rgb32f};
 use serde::{Deserialize, Serialize};
 
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -114,6 +114,20 @@ pub fn demosaic(n: &Normalized, method: Method) -> Rgb32f {
     }
 }
 
+/// [`demosaic`] with [`Method::Ahd`] of the window `r` alone, for Bayer mosaics (`None` for any other data, or a
+/// window not inside the mosaic): the same pixels as demosaicing it all and cropping that to `r`, without
+/// demosaicing the rest or copying the window out of it.
+pub(crate) fn ahd_window(n: &Normalized, r: Rect) -> Option<Rgb32f> {
+    let (w, h) = (n.width, n.height);
+    let cfa = n.cfa.as_ref()?;
+    let inside = r.x.checked_add(r.width).is_some_and(|e| e <= w) && r.y.checked_add(r.height).is_some_and(|e| e <= h);
+    if n.cpp != 1 || !cfa.is_bayer() || w < 4 || h < 4 || n.data.len() != w * h || r.width == 0 || r.height == 0 || !inside {
+        return None;
+    }
+    let m = Mosaic { w, h, data: &n.data, cfa };
+    Some(ahd::ahd_window(&m, r.x, r.y, r.width, r.height))
+}
+
 /// Build a mosaic from an RGB image by sampling each pixel's CFA colour (for tests and synthetic data).
 pub fn mosaic_from_rgb(img: &Rgb32f, cfa: &Cfa) -> Normalized {
     let data = (0..img.height).flat_map(|y| (0..img.width).map(move |x| (x, y))).map(|(x, y)| img.get(x, y)[cfa.color_at(x, y) as usize]).collect();
@@ -192,6 +206,26 @@ mod tests {
         assert!(out.data.iter().all(|p| (p[0] - 0.3).abs() < 1e-5 && (p[1] - 0.55).abs() < 1e-5 && (p[2] - 0.8).abs() < 1e-5));
     }
 
+    /// A window demosaiced alone is the same, bit for bit, as that part of the whole picture: at the edges and
+    /// corners (where the mosaic is reflected), in the middle, across tile seams, at every CFA phase.
+    #[test]
+    fn ahd_window_is_the_crop_of_the_whole() {
+        let img = edgy_scene(300, 270);
+        for pat in ["RGGB", "GBRG"] {
+            let n = mosaic_from_rgb(&img, &Cfa::bayer(pat).unwrap());
+            let whole = demosaic(&n, Method::Ahd);
+            for r in
+                [Rect::new(0, 0, 300, 270), Rect::new(0, 0, 7, 5), Rect::new(1, 3, 298, 266), Rect::new(121, 130, 140, 9), Rect::new(293, 263, 7, 7)]
+            {
+                let part = ahd_window(&n, r).unwrap();
+                let want = whole.clone().into_crop(r.x, r.y, r.width, r.height);
+                assert_eq!((part.width, part.height), (r.width, r.height));
+                assert!(part.data.iter().zip(&want.data).all(|(a, b)| a.map(f32::to_bits) == b.map(f32::to_bits)), "{pat} {r:?}");
+            }
+            assert!(ahd_window(&n, Rect::new(1, 0, 300, 270)).is_none(), "a window past the mosaic is refused");
+        }
+    }
+
     #[test]
     fn known_samples_are_preserved() {
         let img = edgy_scene(40, 32);

crates/raw/src/demosaic/ahd.rs

@@ -21,19 +21,24 @@ const TILE: usize = 128;
 const MARGIN: usize = 6;
 
 pub(crate) fn ahd(m: &Mosaic) -> Rgb32f {
-    let (w, h) = (m.w, m.h);
+    ahd_window(m, 0, 0, m.w, m.h)
+}
+
+/// The `w × h` pixels at (`x`, `y`) of [`ahd`]'s picture (inside the mosaic), demosaiced alone: every pixel comes
+/// from the mosaic around it, so they are the same as when the whole picture is demosaiced and then cropped.
+pub(crate) fn ahd_window(m: &Mosaic, x: usize, y: usize, w: usize, h: usize) -> Rgb32f {
     let mut out = Rgb32f::new(w, h);
     if w == 0 || h == 0 {
         return out;
     }
     // one row of tiles per task, its tiles in parallel too, written straight into the picture
     out.data.par_chunks_mut((w * TILE).max(1)).enumerate().for_each(|(ty, band)| {
-        let (y0, th) = (ty * TILE, band.len() / w.max(1));
+        let (y0, th) = (y + ty * TILE, band.len() / w.max(1));
         let tiles: Vec<(usize, usize, Vec<[f32; 3]>)> = (0..w.div_ceil(TILE))
             .into_par_iter()
             .map(|tx| {
                 let (x0, tw) = (tx * TILE, TILE.min(w - tx * TILE));
-                (x0, tw, tile(m, x0, y0, tw, th))
+                (x0, tw, tile(m, x + x0, y0, tw, th))
             })
             .collect();
         for (x0, tw, px) in tiles {

crates/raw/src/lib.rs

@@ -866,6 +866,15 @@ impl RawImage {
     /// The result is camera RGB (not white balanced), white level = 1.0, not oriented.
     pub fn develop(&self, method: Method) -> Result<Rgb32f> {
         let n = self.normalized()?;
+        // with no `OpcodeList3` to apply to the whole picture first, AHD demosaics just the default crop (the same
+        // pixels, without demosaicing the margin and then copying the crop out of the full picture)
+        if self.opcodes.list3.is_empty() && method == Method::Ahd {
+            let c = self.develop_crop(n.width, n.height);
+            if let Some(rgb) = demosaic::ahd_window(&n, c) {
+                drop(n);
+                return Ok(self.stretched(rgb));
+            }
+        }
         let rgb = demosaic(&n, method);
         drop(n);
         Ok(self.finish_demosaiced(rgb))
@@ -878,6 +887,11 @@ impl RawImage {
         let c = self.develop_crop(rgb.width, rgb.height);
         let rgb =
             if c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height { rgb } else { rgb.into_crop(c.x, c.y, c.width, c.height) };
+        self.stretched(rgb)
+    }
+
+    /// The cropped camera RGB with its rows stretched for non-square pixels.
+    fn stretched(&self, rgb: Rgb32f) -> Rgb32f {
         let h = stretched_height(rgb.height, self.pixel_aspect);
         if h == rgb.height {
             return rgb;