D2: file dialog sheets go on the main window; floating OS windows are blocked while one is up

On macOS rfd put an unparented sheet on AppKit's main window, which after a
click in a floating panel's OS window is that window. With the sheet holding
the keyboard no app window was focused, so craft-ui hid the floating window,
the sheet ended without answering, and later sheets queued behind it: Save As
flashed and closed, and Cmd-S did nothing after. The commands' dialogs now
open at the end of the frame with the main window as parent (macOS; Linux's
portal dialog stays unparented). While one is up the dock area blocks
floating OS windows, as for a modal dialog, and a dialog that ends without
answering never leaves them blocked.

apps/effectcraft/src/main.rs

@@ -128,14 +128,24 @@ fn control_port(name: &str, value: Option<&str>) -> Result<u16, String> {
 /// answer without pumping events, so the window stops answering the compositor and the desktop
 /// offers to force-quit it while the Open dialog is up (#1275). Windows' blocking dialog pumps
 /// its own messages, so it keeps [`effectcraft_ui_egui::Hooks::pick_files`] and friends.
-fn install_file_dialogs(app: &mut EffectcraftApp, ctx: &egui::Context) {
+/// The dialogs asked for in a frame show at its end ([`sheet_dialogs::show_asked`]), where the
+/// main window's handle is at hand.
+fn install_file_dialogs(app: &mut EffectcraftApp) -> Option<std::sync::mpsc::Receiver<sheet_dialogs::Asked>> {
     #[cfg(any(target_os = "macos", target_os = "linux"))]
     {
-        let ctx = ctx.clone();
-        app.hooks.file_dialog_async = Some(Box::new(move |dialog, done| sheet_dialogs::show(dialog, done, ctx.clone())));
+        let (tx, rx) = std::sync::mpsc::channel::<sheet_dialogs::Asked>();
+        app.hooks.file_dialog_async = Some(Box::new(move |dialog, done| {
+            if tx.send((dialog, done)).is_err() {
+                log::error!("file dialog: the window is gone");
+            }
+        }));
+        Some(rx)
     }
     #[cfg(not(any(target_os = "macos", target_os = "linux")))]
-    let _ = (app, ctx);
+    {
+        let _ = app;
+        None
+    }
 }
 
 fn main() -> eframe::Result {
@@ -319,7 +329,7 @@ fn main() -> eframe::Result {
             }));
             // The commands' file dialogs don't block the frame where the platform's backend
             // would (#511, #1275); see [`install_file_dialogs`].
-            install_file_dialogs(&mut app, &cc.egui_ctx);
+            let file_dialogs = install_file_dialogs(&mut app);
             #[cfg(target_os = "macos")]
             {
                 app.hooks.app_action = Some(Box::new(native_menu::app_action));
@@ -337,6 +347,7 @@ fn main() -> eframe::Result {
                 #[cfg(target_os = "macos")]
                 apple_events: apple_events.connect(&cc.egui_ctx),
                 app,
+                file_dialogs,
                 window: window::Tracker::new(opened, saved_window, control_port.is_none()),
                 launch,
                 drawn: false,
@@ -418,6 +429,8 @@ fn startup_failed(e: &eframe::Error, gl_next: bool) {
 /// The desktop app: the egui app plus, on macOS, the system menu bar.
 struct Desktop {
     app: EffectcraftApp,
+    /// The commands' file dialogs, shown at the end of the frame that asked for them.
+    file_dialogs: Option<std::sync::mpsc::Receiver<sheet_dialogs::Asked>>,
     #[cfg(target_os = "macos")]
     menu: native_menu::NativeBar,
     /// Documents opened from Finder, the Dock or `open` (macOS).
@@ -435,10 +448,16 @@ impl eframe::App for Desktop {
         #[cfg(target_os = "macos")]
         self.menu.update(&mut self.app, ctx);
         self.app.logic(ctx, frame);
+        if let Some(asked) = &self.file_dialogs {
+            sheet_dialogs::show_asked(asked, frame, ctx);
+        }
     }
 
     fn ui(&mut self, ui: &mut egui::Ui, frame: &mut eframe::Frame) {
         self.app.ui(ui, frame);
+        if let Some(asked) = &self.file_dialogs {
+            sheet_dialogs::show_asked(asked, frame, ui.ctx());
+        }
         self.window.frame(ui.ctx(), self.app.session.config.as_deref());
         if !self.drawn {
             self.drawn = true;
@@ -638,10 +657,10 @@ mod tests {
     #[test]
     fn commands_file_dialogs_do_not_block_the_frame() {
         let mut app = effectcraft_ui_egui::EffectcraftApp::new(effectcraft_engine::Session::default());
-        super::install_file_dialogs(&mut app, &egui::Context::default());
+        let asked = super::install_file_dialogs(&mut app);
         #[cfg(any(target_os = "macos", target_os = "linux"))]
-        assert!(app.hooks.file_dialog_async.is_some(), "the commands' file dialogs must not block the frame");
+        assert!(app.hooks.file_dialog_async.is_some() && asked.is_some(), "the commands' file dialogs must not block the frame");
         #[cfg(not(any(target_os = "macos", target_os = "linux")))]
-        assert!(app.hooks.file_dialog_async.is_none(), "the blocking dialog pumps its own messages here");
+        assert!(app.hooks.file_dialog_async.is_none() && asked.is_none(), "the blocking dialog pumps its own messages here");
     }
 }

apps/effectcraft/src/sheet_dialogs.rs

@@ -17,10 +17,24 @@ use std::pin::Pin;
 
 type Answer = Pin<Box<dyn Future<Output = Vec<String>> + Send>>;
 
-/// Show `dialog` (on the main thread: the sheet opens here) and hand its paths to `done` once it
-/// closes, then wake the UI.
-pub fn show(dialog: FileDialog, done: DialogDone, ctx: egui::Context) {
-    let answer = open(dialog);
+/// A command's dialog waiting for [`show_asked`].
+pub type Asked = (FileDialog, DialogDone);
+
+/// Show the dialogs the commands asked for in this frame (on the main thread: a sheet opens
+/// here) and hand each one's paths to its `done` once it closes, then wake the UI.
+///
+/// On macOS the sheet goes on the main window (`frame`'s). Without a parent rfd puts it on
+/// AppKit's main window, which is a floating panel's OS window after a click in one; that
+/// window hides while the app has no focused window, and a sheet whose window goes away ends
+/// without answering, with later sheets queued behind it on the same window.
+pub fn show_asked(asked: &std::sync::mpsc::Receiver<Asked>, frame: &eframe::Frame, ctx: &egui::Context) {
+    while let Ok((dialog, done)) = asked.try_recv() {
+        show(dialog, done, frame, ctx.clone());
+    }
+}
+
+fn show(dialog: FileDialog, done: DialogDone, frame: &eframe::Frame, ctx: egui::Context) {
+    let answer = open(dialog, frame);
     let spawned = std::thread::Builder::new().name("ec-file-dialog".into()).spawn(move || {
         done(pollster::block_on(answer));
         ctx.request_repaint();
@@ -30,6 +44,13 @@ pub fn show(dialog: FileDialog, done: DialogDone, ctx: egui::Context) {
     }
 }
 
+/// A dialog on the main window. Linux keeps the portal's dialog unparented: rfd exports the
+/// window for the portal from the dialog's worker thread there.
+fn parented(frame: &eframe::Frame) -> AsyncFileDialog {
+    let d = AsyncFileDialog::new();
+    if cfg!(target_os = "macos") { d.set_parent(frame) } else { d }
+}
+
 fn paths(files: Option<Vec<FileHandle>>) -> Vec<String> {
     files.unwrap_or_default().into_iter().map(|f| f.path().to_string_lossy().to_string()).collect()
 }
@@ -39,10 +60,10 @@ fn path(file: Option<FileHandle>) -> Vec<String> {
 }
 
 /// The dialog, as the blocking hooks in `main.rs` show it.
-fn open(dialog: FileDialog) -> Answer {
+fn open(dialog: FileDialog, frame: &eframe::Frame) -> Answer {
     match dialog {
         FileDialog::Open { exts, many } => {
-            let d = AsyncFileDialog::new().add_filter("Media", &exts);
+            let d = parented(frame).add_filter("Media", &exts);
             if many {
                 let f = d.pick_files();
                 Box::pin(async move { paths(f.await) })
@@ -52,22 +73,22 @@ fn open(dialog: FileDialog) -> Answer {
             }
         }
         FileDialog::OpenProject => {
-            let f = AsyncFileDialog::new().add_filter("EffectCraft Project", &["ecproj", "ecprojx"]).pick_file();
+            let f = parented(frame).add_filter("EffectCraft Project", &["ecproj", "ecprojx"]).pick_file();
             Box::pin(async move { path(f.await) })
         }
         FileDialog::Folder => {
-            let f = AsyncFileDialog::new().pick_folder();
+            let f = parented(frame).pick_folder();
             Box::pin(async move { path(f.await) })
         }
         FileDialog::Save { default, ext, project } => {
             let at = std::path::Path::new(&default);
             let name = at.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
             let mut d = if project {
-                AsyncFileDialog::new().add_filter("EffectCraft Project", &["ecproj"])
+                parented(frame).add_filter("EffectCraft Project", &["ecproj"])
             } else if ext.is_empty() {
-                AsyncFileDialog::new()
+                parented(frame)
             } else {
-                AsyncFileDialog::new().add_filter(&ext, &[ext.as_str()])
+                parented(frame).add_filter(&ext, &[ext.as_str()])
             };
             // A default with a folder that exists opens the dialog there.
             if let Some(dir) = at.parent().filter(|d| d.is_dir()) {

crates/ui-egui/src/dock_ui.rs

@@ -402,6 +402,14 @@ impl EffectcraftApp {
         }
     }
 
+    /// Whether floating panels' OS windows take no input: a dialog of the app's own or a
+    /// command's OS file dialog is up. egui's `Modal` blocks only the window it is in.
+    pub(crate) fn floats_blocked(&mut self, ctx: &egui::Context) -> bool {
+        let (focused, now) = ctx.input(|i| (i.focused || i.raw.viewports.values().any(|v| v.focused == Some(true)), i.time));
+        let file_dialog = self.file_dialogs.up(focused, now);
+        self.dialog.is_some() || panels::dialogs::modal_open(ctx) || file_dialog
+    }
+
     pub(crate) fn dock_area(&mut self, ui: &mut egui::Ui, body: Rect) {
         let t = self.tokens;
         let ctx = ui.ctx().clone();
@@ -418,11 +426,7 @@ impl EffectcraftApp {
         }
         let style = dock_style(ui, &t);
         let maximized = self.ui.maximized.filter(|m| self.ui.dock.is_docked(*m));
-        let mut area = DockArea::new(area_id())
-            .float_bounds(body)
-            .maximized(maximized)
-            .locked(self.ui.dock.locked)
-            .input_blocked(self.dialog.is_some() || panels::dialogs::modal_open(&ctx));
+        let mut area = DockArea::new(area_id()).float_bounds(body).maximized(maximized).locked(self.ui.dock.locked).input_blocked(self.floats_blocked(&ctx));
         if let Some(panel) = self.pending_panel_raise.take() {
             area = area.raise_panel(panel);
         }

crates/ui-egui/src/lib.rs

@@ -319,6 +319,8 @@ pub struct EffectcraftApp {
     device_lost_close: Option<bool>,
     /// Answers of file dialogs shown without blocking ([`Hooks::file_dialog_async`]).
     pub(crate) dialog_answers: (Sender<DialogAnswer>, Receiver<DialogAnswer>),
+    /// Those dialogs that are up, for blocking floating panels' OS windows meanwhile.
+    pub(crate) file_dialogs: FileDialogsUp,
     /// Commands from the native menu bar.
     pub command_inbox: Option<Receiver<String>>,
     /// Pointer position in comp pixels (Info panel).
@@ -406,6 +408,7 @@ impl EffectcraftApp {
             gpu_failure_applied: None,
             device_lost_close: None,
             dialog_answers: std::sync::mpsc::channel(),
+            file_dialogs: FileDialogsUp::default(),
             command_inbox: None,
             pointer_comp: None,
             dialog_state: Default::default(),
@@ -768,6 +771,7 @@ impl EffectcraftApp {
     /// Run the commands whose non-blocking file dialogs have been answered.
     pub(crate) fn run_dialog_answers(&mut self) {
         while let Ok(answer) = self.dialog_answers.1.try_recv() {
+            self.file_dialogs.answered();
             menus::dialog_answered(self, answer);
         }
     }
@@ -1877,6 +1881,55 @@ impl EffectcraftApp {
     }
 }
 
+/// Non-blocking file dialogs ([`Hooks::file_dialog_async`]) that are up. While one is, the dock
+/// area blocks floating panels' OS windows, as it does for a modal dialog; otherwise craft-ui
+/// takes the app for inactive while the dialog has the keyboard, and hides floating palettes.
+///
+/// A dialog can end without answering: on macOS a sheet whose window goes away ends without its
+/// completion handler. A dialog is taken for gone, so floating windows never stay blocked, once
+/// one of the app's windows has the keyboard again after none had it, or when one has kept it
+/// for [`FileDialogsUp::UNSEEN`] since the dialog was asked for, so the dialog never showed.
+#[derive(Default)]
+pub(crate) struct FileDialogsUp {
+    count: usize,
+    /// When the oldest dialog up was first checked ([`FileDialogsUp::up`]).
+    since: Option<f64>,
+    /// None of the app's windows had the keyboard since then.
+    left: bool,
+}
+
+impl FileDialogsUp {
+    /// Seconds a dialog has to take the keyboard from the app's windows. The desktop portal can
+    /// take a moment to show the first dialog on Linux.
+    const UNSEEN: f64 = 2.0;
+
+    pub(crate) fn opened(&mut self) {
+        if self.count == 0 {
+            *self = Self::default();
+        }
+        self.count = self.count.saturating_add(1);
+    }
+
+    pub(crate) fn answered(&mut self) {
+        self.count = self.count.saturating_sub(1);
+    }
+
+    /// Whether a dialog is up at `now`, `focused`: whether one of the app's windows has the
+    /// keyboard.
+    pub(crate) fn up(&mut self, focused: bool, now: f64) -> bool {
+        if self.count == 0 {
+            return false;
+        }
+        let since = *self.since.get_or_insert(now);
+        if !focused {
+            self.left = true;
+        } else if self.left || now - since >= Self::UNSEEN {
+            *self = Self::default();
+        }
+        self.count > 0
+    }
+}
+
 /// Keys and dropped files from floating panels' OS windows (craft-ui `DockOutput`).
 #[derive(Default)]
 pub(crate) struct ForwardedInput {

crates/ui-egui/src/menus.rs

@@ -1415,6 +1415,7 @@ fn file_dialog(app: &mut EffectcraftApp, id: &str, params: &Value) -> Option<Res
         };
         let tx = app.dialog_answers.0.clone();
         let answer = crate::DialogAnswer { command: id.to_string(), key, params: params.clone(), paths: Vec::new() };
+        app.file_dialogs.opened();
         spawn(
             request,
             Box::new(move |paths| {
@@ -2668,6 +2669,56 @@ mod tests {
         let _ = std::fs::remove_dir_all(&dir);
     }
 
+    /// While a command's OS file dialog is up, floating panels' OS windows are blocked, as for
+    /// a modal dialog; a dialog that ends, answered or not, unblocks them, and Save As opens a
+    /// dialog again. A macOS sheet can end without answering (its window went away): once one of
+    /// the app's windows has the keyboard again, or keeps it as if no dialog showed, floating
+    /// windows are not left blocked.
+    #[test]
+    fn file_dialogs_block_floating_windows_only_while_up() {
+        use std::sync::{Arc, Mutex};
+        let mut app = EffectcraftApp::new(effectcraft_engine::Session::default());
+        let ctx = egui::Context::default();
+        let pending: Arc<Mutex<Vec<crate::DialogDone>>> = Arc::default();
+        let queue = pending.clone();
+        app.hooks.file_dialog_async = Some(Box::new(move |_dialog, done| queue.lock().unwrap().push(done)));
+        let blocked = |app: &mut EffectcraftApp, focused: bool, time: f64| {
+            let input = egui::RawInput { focused, time: Some(time), ..Default::default() };
+            let mut out = false;
+            ctx.run_ui(input, |ui| out = app.floats_blocked(ui.ctx())).textures_delta.clear();
+            out
+        };
+        assert!(!blocked(&mut app, true, 0.0));
+        // Cancelled.
+        invoke(&mut app, &ctx, "file.saveAs", json!({})).unwrap();
+        assert!(blocked(&mut app, true, 0.1), "blocked while the dialog takes the keyboard");
+        assert!(blocked(&mut app, false, 0.2));
+        let done = pending.lock().unwrap().pop().expect("a dialog was shown");
+        done(Vec::new());
+        app.run_dialog_answers();
+        assert!(!blocked(&mut app, false, 0.3));
+        // Save As asks again; the sheet ends without answering.
+        invoke(&mut app, &ctx, "file.saveAs", json!({})).unwrap();
+        assert_eq!(pending.lock().unwrap().len(), 1, "Save As opens a dialog again");
+        assert!(blocked(&mut app, true, 1.0));
+        assert!(blocked(&mut app, false, 1.1));
+        assert!(blocked(&mut app, false, 9.0), "a dialog may stay up for long");
+        assert!(!blocked(&mut app, true, 9.1), "an app window has the keyboard again");
+        // A dialog that never takes the keyboard.
+        invoke(&mut app, &ctx, "file.saveAs", json!({})).unwrap();
+        assert!(blocked(&mut app, true, 10.0));
+        assert!(blocked(&mut app, true, 11.0));
+        assert!(!blocked(&mut app, true, 12.5));
+        // Answers of dialogs taken for gone arrive late: nothing stays blocked.
+        for done in pending.lock().unwrap().drain(..) {
+            done(Vec::new());
+        }
+        app.run_dialog_answers();
+        assert!(!blocked(&mut app, false, 13.0));
+        invoke(&mut app, &ctx, "file.saveAs", json!({})).unwrap();
+        assert!(blocked(&mut app, false, 13.1), "a later dialog blocks again");
+    }
+
     #[test]
     fn tga_is_available_in_import_replace_and_proxy_dialogs() {
         use std::sync::{Arc, Mutex};