Issue #1764: running out of device memory falls back without stopping the GPU

An upload whose buffer couldn't be allocated tried again through create_buffer_init, which maps the new buffer and panics inside wgpu when that allocation fails too ("Failed to get mapped range for buffer created with mapped_at_creation"). The panic stopped GPU use for the session. Uploads and kernel parameters now go through the queue into an unmapped buffer, and a failed upload is not retried, so out of memory stays an out-of-memory fallback to the CPU for that render.

Claude-Session: https://claude.ai/code/session_01SF8KiKFYZpuGzdTuzsm3cr

crates/gpu/src/ctx.rs

@@ -3,8 +3,6 @@
 use std::collections::HashMap;
 use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
 
-use wgpu::util::DeviceExt;
-
 /// A kernel module: WGSL source, its storage bindings (name, writable, element type) after the
 /// parameter block `P` (binding 0), and its entry points.
 struct Module {
@@ -520,13 +518,37 @@ impl Gpu {
                 }
                 // (dropping the view schedules the copy)
                 drop(view);
-                return Buf { buf: Some(Tracked::new(buf)), len };
             }
+            // no view: the buffer couldn't be allocated (out of device memory). The error scope has recorded why and
+            // the render fails over to the CPU; allocating it again would only fail again (issue #1764)
+            return Buf { buf: Some(Tracked::new(buf)), len };
         }
-        let buf = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: None, contents: bytes, usage });
+        let buf = self.init_buffer(bytes, usage);
         Buf { buf: Some(Tracked::new(buf)), len }
     }
 
+    /// A buffer holding `contents` (zero-padded to the copy alignment), written through the queue. Not
+    /// `create_buffer_init`: it maps the new buffer, and mapping one that couldn't be allocated (out of device
+    /// memory) panics inside wgpu instead of raising an error the render's scopes catch (issue #1764).
+    fn init_buffer(&self, contents: &[u8], usage: wgpu::BufferUsages) -> wgpu::Buffer {
+        let align = wgpu::COPY_BUFFER_ALIGNMENT as usize;
+        let size = contents.len().max(align).next_multiple_of(align);
+        let buf = self.device.create_buffer(&wgpu::BufferDescriptor {
+            label: None,
+            size: size as u64,
+            usage: usage | wgpu::BufferUsages::COPY_DST,
+            mapped_at_creation: false,
+        });
+        if size == contents.len() {
+            self.queue.write_buffer(&buf, 0, contents);
+        } else {
+            let mut padded = contents.to_vec();
+            padded.resize(size, 0);
+            self.queue.write_buffer(&buf, 0, &padded);
+        }
+        buf
+    }
+
     fn over_limit(&self, size: u64) {
         fail(
             FailKind::Limit,
@@ -570,11 +592,7 @@ impl Gpu {
             return;
         };
         assert_eq!(bufs.len(), k.nbuf, "{name}: binding count");
-        let params = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor {
-            label: None,
-            contents: bytemuck::cast_slice(if p.is_empty() { &[0u32] } else { p }),
-            usage: wgpu::BufferUsages::STORAGE,
-        });
+        let params = self.init_buffer(bytemuck::cast_slice(if p.is_empty() { &[0u32] } else { p }), wgpu::BufferUsages::STORAGE);
         let mut entries = vec![wgpu::BindGroupEntry { binding: 0, resource: params.as_entire_binding() }];
         for (i, b) in bufs.iter().enumerate() {
             let buf = b.map(|b| b.raw()).unwrap_or(&self.dummy);
@@ -902,6 +920,65 @@ mod tests {
         println!("CHILD: survived");
     }
 
+    /// A render that runs out of device memory falls back to the CPU and leaves the GPU in use: the upload
+    /// whose buffer couldn't be allocated used to try again through `create_buffer_init`, which panicked
+    /// mapping the second failed buffer, and the panic stopped GPU use for the session (issue #1764).
+    #[test]
+    fn running_out_of_device_memory_falls_back_without_stopping_the_gpu() {
+        if run_child("out_of_memory_child").is_none() {
+            lightcraft_testkit::skip!(Gpu, "no GPU device, or its memory never ran out");
+        }
+    }
+
+    #[test]
+    #[ignore = "run in its own process by running_out_of_device_memory_falls_back_without_stopping_the_gpu"]
+    fn out_of_memory_child() {
+        let Some(g) = crate::device() else {
+            println!("CHILD: no GPU");
+            return;
+        };
+        // hold device memory until an allocation fails: 1 GiB at a time, then smaller, so little is left
+        let mut held = Vec::new();
+        let mut full = false;
+        for size in [1u64 << 30, 1 << 28, 1 << 26] {
+            for _ in 0..128 {
+                let scope = g.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory);
+                let usage = wgpu::BufferUsages::STORAGE;
+                let buf = g.device.create_buffer(&wgpu::BufferDescriptor { label: None, size, usage, mapped_at_creation: false });
+                if pollster::block_on(scope.pop()).is_some() {
+                    full = true;
+                    break;
+                }
+                held.push(buf);
+            }
+        }
+        if !full {
+            println!("CHILD: no GPU (its memory never ran out)");
+            return;
+        }
+        // a render that needs more than is left: a larger one when the driver still finds room for the first
+        let s = lightcraft_develop::DevelopSettings::default();
+        let mut ran_out = false;
+        for (w, h) in [(4000, 3000), (8000, 6000)] {
+            let src = std::sync::Arc::new(lightcraft_raster::Rgb32f { width: w, height: h, data: vec![[0.18, 0.18, 0.18]; w * h] });
+            let req = lightcraft_pipeline::RenderRequest::fit(w, h);
+            match crate::try_render(&src, &lightcraft_pipeline::SourceInfo::default(), &s, &req, None) {
+                Ok(_) => continue,
+                Err(crate::Fallback::Failed(reason)) => assert!(reason.contains("out of device memory"), "{reason}"),
+                Err(crate::Fallback::NotTried) => panic!("the GPU wasn't tried"),
+            }
+            ran_out = true;
+            break;
+        }
+        assert_eq!(crate::unavailable_reason(), None, "the GPU stays in use");
+        drop(held);
+        if !ran_out {
+            println!("CHILD: no GPU (the renders found room anyway)");
+            return;
+        }
+        println!("CHILD: survived");
+    }
+
     /// Empties the buffer pool when dropped, as the UI does (`trim_pool(0)`) and as returning a
     /// buffer to a full pool does.
     struct TrimOnDrop(&'static super::Gpu);