diff --git a/crates/gpu/src/ctx.rs b/crates/gpu/src/ctx.rs index 7bbbbc68d..b47dab4a0 100644 --- a/crates/gpu/src/ctx.rs +++ b/crates/gpu/src/ctx.rs @@ -3,8 +3,6 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use wgpu::util::DeviceExt; - /// A kernel module: WGSL source, its storage bindings (name, writable, element type) after the /// parameter block `P` (binding 0), and its entry points. struct Module { @@ -520,13 +518,37 @@ impl Gpu { } // (dropping the view schedules the copy) drop(view); - return Buf { buf: Some(Tracked::new(buf)), len }; } + // no view: the buffer couldn't be allocated (out of device memory). The error scope has recorded why and + // the render fails over to the CPU; allocating it again would only fail again (issue #1764) + return Buf { buf: Some(Tracked::new(buf)), len }; } - let buf = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { label: None, contents: bytes, usage }); + let buf = self.init_buffer(bytes, usage); Buf { buf: Some(Tracked::new(buf)), len } } + /// A buffer holding `contents` (zero-padded to the copy alignment), written through the queue. Not + /// `create_buffer_init`: it maps the new buffer, and mapping one that couldn't be allocated (out of device + /// memory) panics inside wgpu instead of raising an error the render's scopes catch (issue #1764). + fn init_buffer(&self, contents: &[u8], usage: wgpu::BufferUsages) -> wgpu::Buffer { + let align = wgpu::COPY_BUFFER_ALIGNMENT as usize; + let size = contents.len().max(align).next_multiple_of(align); + let buf = self.device.create_buffer(&wgpu::BufferDescriptor { + label: None, + size: size as u64, + usage: usage | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + if size == contents.len() { + self.queue.write_buffer(&buf, 0, contents); + } else { + let mut padded = contents.to_vec(); + padded.resize(size, 0); + self.queue.write_buffer(&buf, 0, &padded); + } + buf + } + fn over_limit(&self, size: u64) { fail( FailKind::Limit, @@ -570,11 +592,7 @@ impl Gpu { return; }; assert_eq!(bufs.len(), k.nbuf, "{name}: binding count"); - let params = self.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { - label: None, - contents: bytemuck::cast_slice(if p.is_empty() { &[0u32] } else { p }), - usage: wgpu::BufferUsages::STORAGE, - }); + let params = self.init_buffer(bytemuck::cast_slice(if p.is_empty() { &[0u32] } else { p }), wgpu::BufferUsages::STORAGE); let mut entries = vec![wgpu::BindGroupEntry { binding: 0, resource: params.as_entire_binding() }]; for (i, b) in bufs.iter().enumerate() { let buf = b.map(|b| b.raw()).unwrap_or(&self.dummy); @@ -902,6 +920,65 @@ mod tests { println!("CHILD: survived"); } + /// A render that runs out of device memory falls back to the CPU and leaves the GPU in use: the upload + /// whose buffer couldn't be allocated used to try again through `create_buffer_init`, which panicked + /// mapping the second failed buffer, and the panic stopped GPU use for the session (issue #1764). + #[test] + fn running_out_of_device_memory_falls_back_without_stopping_the_gpu() { + if run_child("out_of_memory_child").is_none() { + lightcraft_testkit::skip!(Gpu, "no GPU device, or its memory never ran out"); + } + } + + #[test] + #[ignore = "run in its own process by running_out_of_device_memory_falls_back_without_stopping_the_gpu"] + fn out_of_memory_child() { + let Some(g) = crate::device() else { + println!("CHILD: no GPU"); + return; + }; + // hold device memory until an allocation fails: 1 GiB at a time, then smaller, so little is left + let mut held = Vec::new(); + let mut full = false; + for size in [1u64 << 30, 1 << 28, 1 << 26] { + for _ in 0..128 { + let scope = g.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory); + let usage = wgpu::BufferUsages::STORAGE; + let buf = g.device.create_buffer(&wgpu::BufferDescriptor { label: None, size, usage, mapped_at_creation: false }); + if pollster::block_on(scope.pop()).is_some() { + full = true; + break; + } + held.push(buf); + } + } + if !full { + println!("CHILD: no GPU (its memory never ran out)"); + return; + } + // a render that needs more than is left: a larger one when the driver still finds room for the first + let s = lightcraft_develop::DevelopSettings::default(); + let mut ran_out = false; + for (w, h) in [(4000, 3000), (8000, 6000)] { + let src = std::sync::Arc::new(lightcraft_raster::Rgb32f { width: w, height: h, data: vec![[0.18, 0.18, 0.18]; w * h] }); + let req = lightcraft_pipeline::RenderRequest::fit(w, h); + match crate::try_render(&src, &lightcraft_pipeline::SourceInfo::default(), &s, &req, None) { + Ok(_) => continue, + Err(crate::Fallback::Failed(reason)) => assert!(reason.contains("out of device memory"), "{reason}"), + Err(crate::Fallback::NotTried) => panic!("the GPU wasn't tried"), + } + ran_out = true; + break; + } + assert_eq!(crate::unavailable_reason(), None, "the GPU stays in use"); + drop(held); + if !ran_out { + println!("CHILD: no GPU (the renders found room anyway)"); + return; + } + println!("CHILD: survived"); + } + /// Empties the buffer pool when dropped, as the UI does (`trim_pool(0)`) and as returning a /// buffer to a full pool does. struct TrimOnDrop(&'static super::Gpu);