From 1c06a793a541fbe0ceae6f280b342fad1ce976ca Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 11 Oct 2026 12:39:23 +0200 Subject: [PATCH] #766: deliver ScriptUI mouse events to script listeners Problem: ScriptUI controls never sent mouse events to scripts. addEventListener("mousedown" | "mouseup" | "mousemove" | "mouseover" | "mouseout" | "click") listeners never ran, so custom-drawn controls (onDraw + mouse, as DuScriptUI builds them) could not react to the pointer. docs/plugins.md listed this as missing. Fix: scriptui.js builds a ScriptUI MouseEvent (clientX/clientY relative to the target, button, detail = click count, screenX/Y, modifier keys, getModifierState, stopPropagation, preventDefault) and delivers it in the capture, target and bubble phases. addEventListener takes the capture flag. The parent walk is bounded, so a script-made parent cycle cannot hang it. Each published control lists the mouse events a listener wants (Widget::mouse, inherited from containers). The new scriptui.mouse command dispatches one event with clamped, finite values and rejects unknown types and disabled controls. The egui ScriptUI view hit-tests the pointer over script windows and docked panels, skipping covered layers and windows blocked by a modal dialog. It sends mouseover/mouseout, mousemove, mousedown/mouseup with click counts, and a click carrying the pointer details in place of the plain click, so a checkbox still toggles once. Handlers run under the existing script limits; a throwing listener reports an error and the window keeps working. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/commands/scripts.rs | 9 ++ crates/engine/src/scriptui.rs | 49 +++++- crates/script/src/lib.rs | 2 + crates/script/src/scriptui.js | 115 +++++++++++++- crates/script/src/tests_ui_mouse.rs | 167 +++++++++++++++++++++ crates/ui-egui/src/panels/scriptui_view.rs | 146 +++++++++++++++++- crates/ui-egui/tests/ui_scriptui.rs | 47 ++++++ docs/agents.md | 5 +- docs/plugins.md | 9 +- docs/scripting-parity.md | 2 +- 10 files changed, 536 insertions(+), 15 deletions(-) create mode 100644 crates/script/src/tests_ui_mouse.rs diff --git a/crates/engine/src/commands/scripts.rs b/crates/engine/src/commands/scripts.rs index 198c9bad..3a1384d9 100644 --- a/crates/engine/src/commands/scripts.rs +++ b/crates/engine/src/commands/scripts.rs @@ -409,6 +409,15 @@ pub fn specs() -> Vec { always, scriptui::set ), + cmd!( + "scriptui.mouse", + "Mouse Event on Script Window Control", + [], + None, + "{window?, widget, type: mousedown | mouseup | mousemove | mouseover | mouseout | click (default), x?, y? (relative to the control; default its centre), button? (0 left, 1 middle, 2 right), detail? (click count), screenX?, screenY?, altKey?, ctrlKey?, metaKey?, shiftKey?} → the script's addEventListener listeners on the control and its containers receive a MouseEvent", + always, + scriptui::mouse + ), cmd!( "scriptui.close", "Close Script Window", diff --git a/crates/engine/src/scriptui.rs b/crates/engine/src/scriptui.rs index 88450c99..620a1124 100644 --- a/crates/engine/src/scriptui.rs +++ b/crates/engine/src/scriptui.rs @@ -110,6 +110,10 @@ pub struct Widget { pub read_only: bool, /// Event handlers the script attached (`onClick`, `onChange`, `onChanging`…). pub handlers: Vec, + /// The mouse events ([`MOUSE_EVENTS`]) a script listener receives when they happen on this + /// control: its own `addEventListener` listeners and its containers' (events bubble). + /// Frontends send only these. + pub mouse: Vec, /// The active tab of a tabbed panel (child index). pub active_tab: usize, /// What the control's `onDraw` handler painted (ScriptUIGraphics), in control coordinates. @@ -364,12 +368,17 @@ pub struct ScriptWindow { pub struct ScriptUiEvent { pub window: u32, pub widget: u32, - /// `click`, `change`, `changing` or `close`. + /// `click`, `change`, `changing`, `close`, or a mouse event ([`MOUSE_EVENTS`]). pub kind: String, - /// The new value: text, number, bool, or selected index(es); for `close`, the result. + /// The new value: text, number, bool, or selected index(es); for `close`, the result; for + /// mouse events (and a mouse `click`), the pointer: `{x, y, button, detail, screenX, screenY, + /// altKey, ctrlKey, metaKey, shiftKey}` with `x`/`y` relative to the control. pub value: Value, } +/// The ScriptUI mouse events scripts can listen for (`addEventListener`). +pub const MOUSE_EVENTS: [&str; 6] = ["mousedown", "mouseup", "mousemove", "mouseover", "mouseout", "click"]; + /// Hands an event to the script that owns the window; returns what the handler reported /// (`{ok, output, error}`) and updates [`Session::script_ui`]. pub type ScriptUiDispatch = fn(&mut Session, &ScriptUiEvent) -> std::result::Result; @@ -687,6 +696,42 @@ pub(crate) fn set(s: &mut Session, p: &Value) -> Result { dispatch(s, ScriptUiEvent { window, widget, kind: kind.into(), value }) } +/// `scriptui.mouse`: a mouse event on a control, as the pointer would make it: the script's +/// `addEventListener` listeners on the control and its containers receive a MouseEvent +/// (`clientX`/`clientY` relative to the control, `button`, `detail` = click count, modifier +/// keys). A `click` also does what clicking the control does (toggle a checkbox, press a button). +pub(crate) fn mouse(s: &mut Session, p: &Value) -> Result { + let c = "scriptui.mouse"; + let window = window_p(s, p, c)?; + let widget = widget_p(s, window, p, c)?; + let ty = p.get("type").and_then(Value::as_str).unwrap_or("click").to_ascii_lowercase(); + let Some(ty) = MOUSE_EVENTS.iter().find(|t| **t == ty) else { + return Err(crate::commands::bad(c, format!("unknown mouse event `{ty}` (one of {})", MOUSE_EVENTS.join(", ")))); + }; + let w = s.script_ui.window(window).and_then(|w| w.root.find(widget)).cloned().unwrap_or_default(); + if !w.enabled { + return Err(crate::commands::bad(c, format!("control {widget} is disabled"))); + } + // Hostile input stays finite and bounded. + const FAR: f64 = 1.0e6; + let num = |k: &str, lo: f64, hi: f64, d: f64| p.get(k).and_then(Value::as_f64).filter(|v| v.is_finite()).map_or(d, |v| v.clamp(lo, hi)); + let flag = |k: &str| p.get(k).and_then(Value::as_bool).unwrap_or(false); + let clicks = if matches!(*ty, "mousemove" | "mouseover" | "mouseout") { 0.0 } else { 1.0 }; + let value = json!({ + "x": num("x", -FAR, FAR, w.bounds[2] / 2.0), + "y": num("y", -FAR, FAR, w.bounds[3] / 2.0), + "screenX": num("screenX", -FAR, FAR, 0.0), + "screenY": num("screenY", -FAR, FAR, 0.0), + "button": num("button", 0.0, 2.0, 0.0).round(), + "detail": num("detail", 0.0, 1000.0, clicks).round(), + "altKey": flag("altKey"), + "ctrlKey": flag("ctrlKey"), + "metaKey": flag("metaKey"), + "shiftKey": flag("shiftKey"), + }); + dispatch(s, ScriptUiEvent { window, widget, kind: (*ty).to_string(), value }) +} + /// `scriptui.close`: close a window (a dialog's `show()` returns `result`, default 2 = Cancel). pub(crate) fn close(s: &mut Session, p: &Value) -> Result { let window = window_p(s, p, "scriptui.close")?; diff --git a/crates/script/src/lib.rs b/crates/script/src/lib.rs index d1c9568c..cc38fe7f 100644 --- a/crates/script/src/lib.rs +++ b/crates/script/src/lib.rs @@ -90,3 +90,5 @@ mod tests_panels; mod tests_ui; #[cfg(test)] mod tests_ui_more; +#[cfg(test)] +mod tests_ui_mouse; diff --git a/crates/script/src/scriptui.js b/crates/script/src/scriptui.js index 8c1c09ed..af1e14b5 100644 --- a/crates/script/src/scriptui.js +++ b/crates/script/src/scriptui.js @@ -594,13 +594,21 @@ function __uiProto() { if (e.indexOf("on") !== 0) e = "on" + e.charAt(0).toUpperCase() + e.slice(1); this.__fire(e); }, - addEventListener: function (name, fn) { + // `capture`: the listener runs in the capture phase of mouse events (on the way down from + // the window to the target) instead of the bubble phase. + addEventListener: function (name, fn, capture) { var k = String(name).toLowerCase(); + if (capture) { + if (!this.__capture) this.__capture = Object.create(null); + (this.__capture[k] = this.__capture[k] || []).push(fn); + return; + } (this.__listeners[k] = this.__listeners[k] || []).push(fn); }, - removeEventListener: function (name, fn) { + removeEventListener: function (name, fn, capture) { var k = String(name).toLowerCase(); - if (this.__listeners[k]) this.__listeners[k] = this.__listeners[k].filter(function (f) { return f !== fn; }); + var m = capture ? this.__capture : this.__listeners; + if (m && m[k]) m[k] = m[k].filter(function (f) { return f !== fn; }); }, dispatchEvent: function (e) { this.__fire("on" + String(e.type || e).charAt(0).toUpperCase() + String(e.type || e).slice(1)); }, __fire: function (name) { @@ -715,7 +723,91 @@ function __uiSize(d) { return [w > 0 ? w : 0, h > 0 ? h : 0]; } -function __uiJson(c) { +// ScriptUI mouse events: delivered to `addEventListener` listeners of the control under the +// pointer and its containers (capture, target, bubble), as the JavaScript Tools Guide documents. +var __uiMouseKinds = ["mousedown", "mouseup", "mousemove", "mouseover", "mouseout", "click"]; + +function __uiMouseListeners(c, type, capture) { + var m = capture ? c.__capture : c.__listeners; + if (!m || !Object.prototype.hasOwnProperty.call(m, type)) return []; + var l = m[type]; + return l instanceof Array ? l.slice() : []; +} + +// The mouse events that reach a listener at `c`: its own, and its containers' (`inherited`). +function __uiMouseWanted(c, inherited) { + var out = inherited.slice(); + for (var i = 0; i < __uiMouseKinds.length; i++) { + var t = __uiMouseKinds[i]; + if (out.indexOf(t) < 0 && (__uiMouseListeners(c, t, false).length || __uiMouseListeners(c, t, true).length)) out.push(t); + } + return out; +} + +// A ScriptUI MouseEvent. `v`: {x, y (relative to the target control), button (0 left, 1 middle, +// 2 right), detail (click count), screenX, screenY, altKey, ctrlKey, metaKey, shiftKey}. +function __uiMouseEvent(type, target, v) { + var num = function (x, d) { + x = Number(x); + return isFinite(x) ? x : d; + }; + var keys = { alt: !!v.altKey, control: !!v.ctrlKey, ctrl: !!v.ctrlKey, meta: !!v.metaKey, shift: !!v.shiftKey }; + return { + type: type, + target: target, + currentTarget: target, + eventPhase: "none", + bubbles: true, + cancelable: true, + captures: true, + view: target.window, + relatedTarget: undefined, + timeStamp: new Date(), + detail: num(v.detail, type === "mousemove" || type === "mouseover" || type === "mouseout" ? 0 : 1), + button: num(v.button, 0), + clientX: num(v.x, 0), + clientY: num(v.y, 0), + screenX: num(v.screenX, 0), + screenY: num(v.screenY, 0), + altKey: keys.alt, + ctrlKey: keys.ctrl, + metaKey: keys.meta, + shiftKey: keys.shift, + defaultPrevented: false, + __stopped: false, + stopPropagation: function () { this.__stopped = true; }, + preventDefault: function () { this.defaultPrevented = true; }, + getModifierState: function (k) { return !!keys[String(k).toLowerCase()]; }, + toString: function () { return "[object MouseEvent]"; } + }; +} + +// Deliver a mouse event to `target`: capture listeners from the window down, the target's own +// listeners, then bubble listeners back up. stopPropagation() ends it after the current control. +function __uiMouseFire(target, type, v) { + var path = []; + // A tree built by add() is shallow; the bound keeps a script's own `parent` cycle harmless. + for (var p = target; p && path.length < 256; p = p.parent) path.unshift(p); + var e = __uiMouseEvent(type, target, v); + var run = function (c, phase, capture) { + var l = __uiMouseListeners(c, type, capture); + e.currentTarget = c; + e.eventPhase = phase; + for (var i = 0; i < l.length; i++) if (typeof l[i] === "function") l[i].call(c, e); + }; + var n = path.length - 1; + for (var i = 0; i < n; i++) { + run(path[i], "capture", true); + if (e.__stopped) return e; + } + run(target, "target", true); + run(target, "target", false); + for (var j = n - 1; j >= 0 && !e.__stopped; j--) run(path[j], "bubble", false); + return e; +} + +function __uiJson(c, inherited) { + var mouse = __uiMouseWanted(c, inherited || []); var ps = __uiSize(c.preferredSize); if (c.type === "edittext" && c.characters > 0) ps = [c.characters * 7 + 12, ps ? ps[1] : 0]; var o = { @@ -730,6 +822,7 @@ function __uiJson(c) { children: [] }; if (ps) o.preferredSize = ps; + if (mouse.length) o.mouse = mouse; if (c.__ub && !c.__root) o.fixedBounds = c.__ub; if (c.__root && c.__ub) o.fixedBounds = [0, 0, c.__ub[2] - c.__ub[0], c.__ub[3] - c.__ub[1]]; if (__uiContainers[c.type]) { @@ -739,7 +832,7 @@ function __uiJson(c) { if (m) o.margins = m; if (typeof c.spacing === "number") o.spacing = c.spacing; var kids = c.children; - for (var i = 0; i < kids.length; i++) o.children.push(__uiJson(kids[i])); + for (var i = 0; i < kids.length; i++) o.children.push(__uiJson(kids[i], mouse)); } if (c.alignment !== null && c.alignment !== undefined) o.alignment = __uiAlign(c.alignment); if (c.type === "checkbox" || c.type === "radiobutton") o.checked = !!c.value; @@ -824,16 +917,26 @@ function __uiDispatch(winId, id, kind, valueJson) { for (var i = 0; i < sib.length; i++) if (sib[i] !== c && sib[i].type === "radiobutton") sib[i].value = false; c.value = true; }; + // A mouse click carries the pointer's details ({x, y, button, detail, …keys}). + var mouse = v !== null && typeof v === "object" && !(v instanceof Array) ? v : null; + if (kind !== "click" && __uiMouseKinds.indexOf(kind) >= 0) { + __uiMouseFire(c, kind, mouse || {}); + return; + } if (kind === "click") { if (c.type === "checkbox") c.value = !c.value; else if (c.type === "radiobutton") radio(); else if (c.type === "tab" && c.parent) { c.parent.selection = c; c.parent.__fire("onChange"); + if (mouse) __uiMouseFire(c, "click", mouse); return; } var handled = c.__handlers().indexOf("onClick") >= 0; - c.__fire("onClick"); + if (mouse) { + if (typeof c.onClick === "function") c.onClick.call(c); + __uiMouseFire(c, "click", mouse); + } else c.__fire("onClick"); // Dialog buttons without handlers: OK closes with 1, Cancel with 2. if (!handled && c.type === "button" && w.__kind === "dialog") { var n = String(c.name || c.properties.name || c.text).toLowerCase(); diff --git a/crates/script/src/tests_ui_mouse.rs b/crates/script/src/tests_ui_mouse.rs new file mode 100644 index 00000000..ddacac01 --- /dev/null +++ b/crates/script/src/tests_ui_mouse.rs @@ -0,0 +1,167 @@ +//! ScriptUI mouse events (#766): `addEventListener("mousedown" | "mouseup" | "mousemove" | +//! "mouseover" | "mouseout" | "click", handler)` listeners receive a MouseEvent through +//! `scriptui.mouse`, with capture / target / bubble phases as the JavaScript Tools Guide +//! documents. + +use std::sync::Arc; + +use effectcraft_engine::Session; +use serde_json::{Value, json}; + +use crate::run_code; + +fn session() -> Session { + let mut s = Session { expr: Some(Arc::new(effectcraft_expr::Expressions)), ..Default::default() }; + crate::install(&mut s); + s.config = Some(Arc::new(effectcraft_engine::config::MemoryConfig::default())); + s +} + +const CANVAS: &str = r#" +var w = new Window("palette", "Canvas"); +var g = w.add("group", undefined, undefined, { name: "row" }); +var pad = g.add("group", undefined, undefined, { name: "pad" }); +pad.preferredSize = [120, 80]; +var box = w.add("checkbox", undefined, "Snap", { name: "snap" }); +var plain = w.add("statictext", undefined, "no listeners", { name: "plain" }); +function rec(tag) { + return function (e) { + writeLn(tag + ":" + e.type + "@" + e.eventPhase + " " + e.clientX + "," + e.clientY + " b" + e.button + " d" + e.detail + + (e.shiftKey ? " shift" : "") + (e.altKey ? " alt" : "") + (e.getModifierState("Control") ? " ctrl" : "") + + (e.currentTarget === this ? "" : " WRONG-THIS") + " " + e.target.properties.name); + }; +} +pad.addEventListener("mousedown", rec("pad")); +pad.addEventListener("mousemove", rec("pad")); +pad.addEventListener("mouseover", rec("pad")); +pad.addEventListener("mouseout", rec("pad")); +pad.addEventListener("mouseup", rec("pad")); +g.addEventListener("mousedown", rec("row")); +w.addEventListener("mousedown", rec("win"), true); +box.onClick = function () { writeLn("onClick " + box.value); }; +box.addEventListener("click", rec("box")); +w.show(); +"#; + +fn lines(r: &Value) -> Vec { + assert_eq!(r["ok"], true, "{r}"); + r["output"].as_str().unwrap_or_default().lines().map(str::to_string).collect() +} + +#[test] +fn mouse_events_reach_listeners_with_pointer_details() { + let mut s = session(); + let o = run_code(&mut s, CANVAS, "canvas.jsx"); + assert!(o.error.is_none(), "{:?}", o.error); + let w = s.script_ui.windows[0].clone(); + // The published tree says which controls want which mouse events (containers' listeners + // reach their children: events bubble). + let pad = w.root.lookup(&json!("pad")).unwrap(); + assert_eq!(pad.mouse, ["mousedown", "mouseup", "mousemove", "mouseover", "mouseout"]); + assert_eq!(w.root.lookup(&json!("snap")).unwrap().mouse, ["mousedown", "click"], "the window's capture listener + its own"); + assert_eq!(w.root.lookup(&json!("plain")).unwrap().mouse, ["mousedown"]); + // A press: capture on the window, the target, then bubbling up through the group. + let r = s + .execute( + "scriptui.mouse", + json!({"widget": "pad", "type": "mousedown", "x": 12.5, "y": 30, "button": 2, "detail": 2, "shiftKey": true, "ctrlKey": true}), + ) + .unwrap(); + assert_eq!( + lines(&r), + [ + "win:mousedown@capture 12.5,30 b2 d2 shift ctrl pad", + "pad:mousedown@target 12.5,30 b2 d2 shift ctrl pad", + "row:mousedown@bubble 12.5,30 b2 d2 shift ctrl pad", + ] + ); + // Hover and moves carry no click count; x / y default to the control's centre. + let r = s.execute("scriptui.mouse", json!({"widget": "pad", "type": "mouseover"})).unwrap(); + assert_eq!(lines(&r), ["pad:mouseover@target 60,40 b0 d0 pad"]); + let r = s.execute("scriptui.mouse", json!({"widget": "pad", "type": "MouseMove", "x": 3, "y": 4, "altKey": true})).unwrap(); + assert_eq!(lines(&r), ["pad:mousemove@target 3,4 b0 d0 alt pad"]); + // An event nobody listens for on a control runs nothing (but still answers). + let r = s.execute("scriptui.mouse", json!({"widget": "plain", "type": "mouseup"})).unwrap(); + assert!(lines(&r).is_empty()); + // A mouse click does what a click does (the checkbox toggles once, onClick runs) and its + // listeners get the pointer. + let r = s.execute("scriptui.mouse", json!({"widget": "snap", "type": "click", "x": 5, "y": 6})).unwrap(); + assert_eq!(lines(&r), ["onClick true", "box:click@target 5,6 b0 d1 snap"]); + assert!(s.script_ui.windows[0].root.lookup(&json!("snap")).unwrap().checked); + // Agents' plain click still fires the click listeners. + let r = s.execute("scriptui.click", json!({"widget": "snap"})).unwrap(); + assert_eq!(r["output"].as_str().unwrap_or_default().lines().next(), Some("onClick false")); + s.execute("scriptui.close", json!({})).unwrap(); +} + +#[test] +fn stop_propagation_capture_and_removal() { + let mut s = session(); + let code = r#" + var w = new Window("palette", "Stops"); + var g = w.add("group", undefined, undefined, { name: "outer" }); + var b = g.add("button", undefined, "Go", { name: "go" }); + var stopHere = function (e) { writeLn("capture " + e.currentTarget.properties.name); e.stopPropagation(); }; + g.addEventListener("mouseup", stopHere, true); + b.addEventListener("mouseup", function () { writeLn("never"); }); + var bubble = function (e) { writeLn("bubble " + e.eventPhase); }; + g.addEventListener("mousedown", bubble); + b.addEventListener("mousedown", function (e) { writeLn("target"); e.stopPropagation(); e.preventDefault(); writeLn("prevented " + e.defaultPrevented); }); + b.addEventListener("mousemove", function () { writeLn("move"); }); + b.removeEventListener("mousemove", b.__listeners.mousemove[0]); + w.show(); + "#; + let o = run_code(&mut s, code, "stops.jsx"); + assert!(o.error.is_none(), "{:?}", o.error); + // A capture listener that stops: the target never hears it. + let r = s.execute("scriptui.mouse", json!({"widget": "go", "type": "mouseup"})).unwrap(); + assert_eq!(lines(&r), ["capture outer"]); + // Stopped at the target: no bubbling to the group. + let r = s.execute("scriptui.mouse", json!({"widget": "go", "type": "mousedown"})).unwrap(); + assert_eq!(lines(&r), ["target", "prevented true"]); + // The group itself as the target. + let r = s.execute("scriptui.mouse", json!({"widget": "outer", "type": "mousedown"})).unwrap(); + assert_eq!(lines(&r), ["bubble target"]); + // Removed listeners don't run, and no longer ask for the event. + let r = s.execute("scriptui.mouse", json!({"widget": "go", "type": "mousemove"})).unwrap(); + assert!(lines(&r).is_empty()); + assert!(!s.script_ui.windows[0].root.lookup(&json!("go")).unwrap().mouse.iter().any(|m| m == "mousemove")); + s.execute("scriptui.close", json!({})).unwrap(); +} + +/// Hostile input: unknown events, disabled controls, absurd coordinates, a throwing listener, +/// a script that makes a control its own parent. +#[test] +fn mouse_events_survive_bad_input_and_throwing_listeners() { + let mut s = session(); + let code = r#" + var w = new Window("palette", "Bad"); + var b = w.add("button", undefined, "Boom", { name: "boom" }); + var off = w.add("button", undefined, "Off", { name: "off" }); + off.enabled = false; + off.addEventListener("mousedown", function () { writeLn("disabled ran"); }); + var n = 0; + b.addEventListener("mousedown", function (e) { n++; writeLn("at " + e.clientX + "," + e.clientY + " b" + e.button + " d" + e.detail); if (n === 2) throw new Error("bad press"); }); + var loop = w.add("group", undefined, undefined, { name: "loop" }); + loop.addEventListener("mousedown", function () { writeLn("loop ok"); }); + w.show(); + loop.parent = loop; + "#; + let o = run_code(&mut s, code, "bad.jsx"); + assert!(o.error.is_none(), "{:?}", o.error); + assert!(s.execute("scriptui.mouse", json!({"widget": "boom", "type": "dblclick"})).is_err()); + assert!(s.execute("scriptui.mouse", json!({"widget": "off", "type": "mousedown"})).is_err(), "disabled controls get no events"); + let r = s.execute("scriptui.mouse", json!({"widget": "boom", "type": "mousedown", "x": 1e300, "y": -1e300, "button": 99, "detail": -4})).unwrap(); + assert_eq!(lines(&r), ["at 1000000,-1000000 b2 d0"]); + // A throwing listener reports its error; the window keeps working. + let r = s.execute("scriptui.mouse", json!({"widget": "boom", "type": "mousedown"})).unwrap(); + assert_eq!(r["ok"], false, "{r}"); + assert!(r["error"]["message"].as_str().unwrap_or_default().contains("bad press"), "{r}"); + let r = s.execute("scriptui.mouse", json!({"widget": "boom", "type": "mousedown"})).unwrap(); + let at = s.script_ui.windows[0].root.lookup(&json!("boom")).unwrap().bounds; + assert_eq!(lines(&r), [format!("at {},{} b0 d1", at[2] / 2.0, at[3] / 2.0)], "x / y default to the centre"); + // A parent cycle made by the script ends instead of hanging. + let r = s.execute("scriptui.mouse", json!({"widget": "loop", "type": "mousedown"})).unwrap(); + assert!(lines(&r).iter().any(|l| l == "loop ok"), "{r}"); + s.execute("scriptui.close", json!({})).unwrap(); +} diff --git a/crates/ui-egui/src/panels/scriptui_view.rs b/crates/ui-egui/src/panels/scriptui_view.rs index e8793388..f0a3975f 100644 --- a/crates/ui-egui/src/panels/scriptui_view.rs +++ b/crates/ui-egui/src/panels/scriptui_view.rs @@ -4,7 +4,9 @@ //! `scriptui.close` commands, which run the script's handlers. Every control registers an //! automation id `scriptui..` (and `scriptui..` when the script //! named it). Controls with an `onDraw` handler are painted from their draw list; edit texts and -//! sliders send live `changing` updates (onChanging) while typing / dragging. +//! sliders send live `changing` updates (onChanging) while typing / dragging. Mouse events +//! (`mousedown`, `mouseup`, `mousemove`, `mouseover`, `mouseout`, `click`) on a control go to the +//! script through `scriptui.mouse` when a listener wants them (`Widget::mouse`). use crate::i18n::tr; use effectcraft_engine::Services; @@ -22,6 +24,8 @@ enum Act { /// A live update (each keystroke / slider step): fires onChanging. Changing(u32, u32, Value), Close(u32), + /// A mouse event (`scriptui.mouse` parameters without window and widget). + Mouse(u32, u32, Value), } fn rect_of(origin: egui::Pos2, b: [f64; 4]) -> Rect { @@ -335,6 +339,132 @@ fn paint_image(ui: &egui::Ui, r: Rect, img: &ImageRef, services: &dyn Services) true } +/// Where the pointer was and what it did over one script window (for mouseover / mouseout and +/// click counts). +#[derive(Clone, Default)] +struct MouseTrack { + hover: Option, + at: Option, + /// The last press: control, time, position (a quick second press there counts as a double + /// click). + last_down: Option<(u32, f64, egui::Pos2)>, + clicks: u32, + /// The control and button pressed (a release on the same control is a click). + down: Option<(u32, u8)>, +} + +/// The deepest visible control of `w` at `p` (`w` itself when no child is there): later +/// siblings are on top; a tabbed panel shows its active tab only. +fn hit_test(w: &Widget, origin: egui::Pos2, p: egui::Pos2, depth: usize) -> Option<&Widget> { + if !w.visible || !rect_of(origin, w.bounds).contains(p) { + return None; + } + // Widget trees come from scripts: the depth is bounded. + if depth < 64 { + if w.kind == WidgetKind::TabbedPanel { + if let Some(hit) = w.children.get(w.active_tab).and_then(|t| hit_test(t, origin, p, depth + 1)) { + return Some(hit); + } + } else if let Some(hit) = w.children.iter().rev().find_map(|c| hit_test(c, origin, p, depth + 1)) { + return Some(hit); + } + } + Some(w) +} + +fn wants_mouse(w: &Widget, depth: usize) -> bool { + !w.mouse.is_empty() || (depth < 64 && w.children.iter().any(|c| wants_mouse(c, depth + 1))) +} + +/// Mouse events over a script window's controls (drawn at `origin`, visible in `area`) for the +/// script's listeners: mouseover / mouseout when the control under the pointer changes, +/// mousemove, mousedown / mouseup, and click (a press and release of the left button on the +/// same control; it replaces the plain click the control reported, so a checkbox toggles once). +/// Only events a listener wants are sent; none while `active` is false (a modal dialog is up). +fn mouse_events(ui: &egui::Ui, origin: egui::Pos2, area: Rect, win: u32, root: &Widget, active: bool, acts: &mut Vec) { + let id = egui::Id::new(("sui-mouse", win)); + if !wants_mouse(root, 0) { + ui.data_mut(|d| d.remove::(id)); + return; + } + let mut st: MouseTrack = ui.data(|d| d.get_temp(id)).unwrap_or_default(); + let (latest, events, mods, time) = ui.input(|i| (i.pointer.latest_pos(), i.events.clone(), i.modifiers, i.time)); + let ctx = ui.ctx(); + let layer = ui.layer_id(); + let target = |p: egui::Pos2| -> Option<&Widget> { + if !active || !area.contains(p) || ctx.layer_id_at(p).is_some_and(|l| l != layer) { + return None; + } + hit_test(root, origin, p, 0) + }; + let send = |w: &Widget, ty: &str, p: egui::Pos2, button: u8, detail: u32, acts: &mut Vec| { + if !w.enabled || !w.mouse.iter().any(|m| m == ty) { + return; + } + let at = rect_of(origin, w.bounds).min; + acts.push(Act::Mouse( + win, + w.id, + json!({ + "type": ty, "x": p.x - at.x, "y": p.y - at.y, "screenX": p.x, "screenY": p.y, + "button": button, "detail": detail, + "altKey": mods.alt, "ctrlKey": mods.ctrl, "shiftKey": mods.shift, "metaKey": mods.mac_cmd, + }), + )); + }; + // The pointer moved: out of the old control, over the new one, along it. + let now = latest.and_then(|p| target(p).map(|w| (w, p))); + let now_id = now.map(|(w, _)| w.id); + if now_id != st.hover { + if let Some(old) = st.hover.and_then(|h| root.find(h)) { + send(old, "mouseout", latest.unwrap_or(origin), 0, 0, acts); + } + if let Some((w, p)) = now { + send(w, "mouseover", p, 0, 0, acts); + } + st.hover = now_id; + } + if let Some((w, p)) = now + && st.at != Some(p) + { + send(w, "mousemove", p, 0, 0, acts); + } + st.at = latest; + // Presses and releases. + for e in &events { + let egui::Event::PointerButton { pos, button, pressed, .. } = e else { continue }; + let b = match button { + egui::PointerButton::Primary => 0, + egui::PointerButton::Middle => 1, + egui::PointerButton::Secondary => 2, + _ => continue, + }; + let Some(w) = target(*pos) else { + if !pressed { + st.down = None; + } + continue; + }; + if *pressed { + let again = st.last_down.is_some_and(|(c, t, at)| c == w.id && time - t < 0.5 && at.distance(*pos) < 6.0); + st.clicks = if again { st.clicks.saturating_add(1) } else { 1 }; + st.last_down = Some((w.id, time, *pos)); + st.down = Some((w.id, b)); + send(w, "mousedown", *pos, b, st.clicks, acts); + } else { + send(w, "mouseup", *pos, b, st.clicks, acts); + if st.down == Some((w.id, b)) && b == 0 && w.enabled && w.mouse.iter().any(|m| m == "click") { + // The click with the pointer's details, after mousedown / mouseup, in place of + // the plain click the control reported. + acts.retain(|a| !matches!(a, Act::Click(aw, ac) if *aw == win && *ac == w.id)); + send(w, "click", *pos, b, st.clicks, acts); + } + st.down = None; + } + } + ui.data_mut(|d| d.insert_temp(id, st)); +} + /// Checkboxes and radio buttons sit at the left of their (possibly filled) bounds. fn snug(r: Rect, text: &str) -> Rect { let w = (text.chars().count() as f32 * 7.0 + 26.0).min(r.width()); @@ -348,6 +478,13 @@ fn perform(app: &mut EffectcraftApp, ctx: &egui::Context, acts: Vec) { Act::Set(w, id, v) => ("scriptui.set", json!({"window": w, "widget": id, "value": v})), Act::Changing(w, id, v) => ("scriptui.set", json!({"window": w, "widget": id, "value": v, "changing": true})), Act::Close(w) => ("scriptui.close", json!({"window": w})), + Act::Mouse(w, id, mut v) => { + if let Some(o) = v.as_object_mut() { + o.insert("window".into(), json!(w)); + o.insert("widget".into(), json!(id)); + } + ("scriptui.mouse", v) + } }; match crate::menus::invoke(app, ctx, cmd, p) { Ok(r) => { @@ -372,7 +509,8 @@ pub fn show_windows(app: &mut EffectcraftApp, ctx: &egui::Context) { let t = app.tokens; let mut acts = vec![]; let screen = ctx.content_rect(); - if windows.iter().any(|w| w.modal) { + let blocked = windows.iter().any(|w| w.modal); + if blocked { // A modal dialog blocks the app behind it, as in After Effects. let r = egui::Area::new(egui::Id::new("scriptui-modal-block")).order(egui::Order::Middle).fixed_pos(screen.min).show(ctx, |ui| { ui.allocate_exact_size(screen.size(), Sense::click_and_drag()); @@ -395,6 +533,7 @@ pub fn show_windows(app: &mut EffectcraftApp, ctx: &egui::Context) { .show(ctx, |ui| { let (rect, _) = ui.allocate_exact_size(size, Sense::hover()); draw_children(app, ui, rect.min, w.id, &w.root, &mut acts); + mouse_events(ui, rect.min, rect, w.id, &w.root, w.modal || !blocked, &mut acts); rect }); if let Some(r) = resp { @@ -418,11 +557,14 @@ pub fn panel(app: &mut EffectcraftApp, ui: &mut egui::Ui, rect: Rect, id: u32) { let (pw, ph) = (rect.width() as f64, rect.height() as f64); w.root.fixed_bounds = Some([0.0, 0.0, pw.max(w.root.bounds[2]), ph.max(w.root.bounds[3])]); layout(&mut w); + // A modal script dialog blocks the panels too. + let modal = app.session.script_ui.windows.iter().any(|w| w.modal); let mut acts = vec![]; let content = rect; egui::ScrollArea::both().id_salt(("scriptui-panel", id)).max_width(content.width()).max_height(content.height()).show(ui, |ui| { let (r, _) = ui.allocate_exact_size(vec2(w.root.bounds[2] as f32, w.root.bounds[3] as f32), Sense::hover()); draw_children(app, ui, r.min, id, &w.root, &mut acts); + mouse_events(ui, r.min, r.intersect(ui.clip_rect()), id, &w.root, !modal, &mut acts); }); app.auto.add(&format!("scriptui.{id}"), rect, &w.title); let ctx = ui.ctx().clone(); diff --git a/crates/ui-egui/tests/ui_scriptui.rs b/crates/ui-egui/tests/ui_scriptui.rs index 5a01d18f..6cfc2db7 100644 --- a/crates/ui-egui/tests/ui_scriptui.rs +++ b/crates/ui-egui/tests/ui_scriptui.rs @@ -224,3 +224,50 @@ fn on_draw_concave_fills_and_images_render() { h.state_mut().session.execute("scriptui.close", json!({})).unwrap(); let _ = std::fs::remove_dir_all(&dir); } + +/// ScriptUI mouse events (#766): the pointer over a script window reaches the script's +/// `addEventListener` listeners (element-relative position, button, click count), and a +/// pointer click on a checkbox with a click listener toggles it once. +#[test] +fn script_windows_send_mouse_events_to_listeners() { + let mut h = harness(); + let code = r#" + var w = new Window("palette", "Pad"); + var pad = w.add("group", undefined, undefined, { name: "pad" }); + pad.preferredSize = [120, 80]; + var log = w.add("statictext", undefined, "", { name: "log" }); + log.preferredSize = [400, 20]; + var snap = w.add("checkbox", undefined, "Snap", { name: "snap" }); + var seen = []; + var rec = function (e) { seen.push(e.type + " " + Math.round(e.clientX) + "," + Math.round(e.clientY) + " b" + e.button + " d" + e.detail); log.text = seen.join("|"); }; + pad.addEventListener("mouseover", rec); + pad.addEventListener("mousedown", rec); + pad.addEventListener("mouseup", rec); + pad.addEventListener("click", rec); + snap.addEventListener("click", function (e) { log.text = "snap " + snap.value + " d" + e.detail; }); + w.show(); + "#; + let r = h.state_mut().session.execute("script.run", json!({"code": code, "name": "pad.jsx"})).unwrap(); + assert_eq!(r["ok"], true, "{r}"); + h.run_steps(3); + let win = h.state().session.script_ui.windows[0].id; + let text = |h: &Harness<'_, EffectcraftApp>, name: &str| h.state().session.script_ui.windows[0].root.lookup(&json!(name)).cloned().unwrap(); + let pad = rect(&h, &format!("scriptui.{win}.pad")); + let at = pad.min + vec2(10.0, 20.0); + click_at(&mut h, at); + assert_eq!(text(&h, "log").text, "mouseover 10,20 b0 d0|mousedown 10,20 b0 d1|mouseup 10,20 b0 d1|click 10,20 b0 d1"); + click(&mut h, &format!("scriptui.{win}.snap")); + let (snap, log) = (text(&h, "snap"), text(&h, "log")); + assert!(snap.checked, "toggled once"); + assert_eq!(log.text, "snap true d1"); + h.state_mut().session.execute("scriptui.close", json!({})).unwrap(); +} + +fn click_at(h: &mut Harness<'_, EffectcraftApp>, c: egui::Pos2) { + h.input_mut().events.push(Event::PointerMoved(c)); + h.run_steps(1); + for pressed in [true, false] { + h.input_mut().events.push(Event::PointerButton { pos: c, button: egui::PointerButton::Primary, pressed, modifiers: Default::default() }); + } + h.run_steps(3); +} diff --git a/docs/agents.md b/docs/agents.md index 6a3b9e85..583c0219 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -512,7 +512,10 @@ Scripts that build ScriptUI windows publish them to the session; agents drive th 3. `scriptui.click {"widget": "ok"}` presses a button / toggles a checkbox / picks a radio button or tab; `scriptui.set {"widget": "#4", "value": "Shot_"}` types into edit text, moves a slider or picks a list item (index or text); `scriptui.close {"result": 2}` closes. Controls are addressed - by id, `#id`, `properties.name` or text; `window` can be omitted when one window is open. Each + by id, `#id`, `properties.name` or text; `window` can be omitted when one window is open. + `scriptui.mouse {"widget": "pad", "type": "mousedown", "x": 10, "y": 20}` sends a mouse event + (`mousedown`, `mouseup`, `mousemove`, `mouseover`, `mouseout`, `click`; `x`/`y` relative to the + control, `button`, `detail`, `shiftKey`…) to the script's `addEventListener` listeners. Each returns the handler run's `{ok, output, error}`. A dialog's `show()` waits for the user: the `script.run` / `file.runScript` reply carries diff --git a/docs/plugins.md b/docs/plugins.md index d24c078b..41c2eddb 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -84,9 +84,12 @@ Supported controls: `panel`, `group`, `button`, `iconbutton`, `statictext`, `edi `tabbedpanel` / `tab` and `image`, with ScriptUI's automatic layout (`orientation`, `alignChildren`, `alignment`, `margins`, `spacing`, `preferredSize`), resource strings, event handlers (`onClick`, `onChange`, `onChanging`, `onClose`, `addEventListener`) and `onDraw` with -ScriptUIGraphics. Mouse events (`mousedown`, `mousemove`…) aren't sent to scripts yet. Agents use -a panel like a user: `scriptui.list`, `scriptui.get` (its controls), `scriptui.click`, -`scriptui.set` and `scriptui.close`; every control has the automation id +ScriptUIGraphics. Mouse events (`addEventListener("mousedown" | "mouseup" | "mousemove" | +"mouseover" | "mouseout" | "click", handler, capture?)`) reach the control under the pointer and +its containers (capture, target, bubble; `stopPropagation()`), with a MouseEvent carrying +`clientX`/`clientY` relative to the target, `button`, `detail` (click count) and the modifier +keys. Agents use a panel like a user: `scriptui.list`, `scriptui.get` (its controls), +`scriptui.click`, `scriptui.set`, `scriptui.mouse` and `scriptui.close`; every control has the automation id `scriptui..`. The bundled sample `Layer Tools.jsx` (`crates/engine/scripts/ScriptUI Panels/`) and the Ease Presets extension below are longer examples. diff --git a/docs/scripting-parity.md b/docs/scripting-parity.md index fef88efb..c4b0a197 100644 --- a/docs/scripting-parity.md +++ b/docs/scripting-parity.md @@ -19,7 +19,7 @@ our own scripts; no real-world script library has been run against both apps. | Expression editing | Inline field, error bar, pick whip, Expression Language menu, external editors | Inline field that grows, error bar, pick whips (property and value), language menu | partial | External editor support (#364) | | Expression performance | Cached, multi-frame | Per-thread boa contexts | unknown | No benchmark | | Scripting object model | `app`, `Project`, items, layers, properties, keys, eases, render queue, `TextDocument`, `Shape`, markers, `File`/`Folder`, `Socket`, `app.settings` | Same classes, edits run through undoable engine commands; match names table | good | `crates/script` tests (≈ 10 test files) | -| ScriptUI | Dialogs, palettes, dockable panels, resource strings, `onDraw` / ScriptUIGraphics | Same, dockable panels reopen at launch | good | `ui_scriptui`, `tests_ui*` | +| ScriptUI | Dialogs, palettes, dockable panels, resource strings, `onDraw` / ScriptUIGraphics, mouse events | Same, dockable panels reopen at launch | good | `ui_scriptui`, `tests_ui*` | | Scripts menu, ScriptUI Panels folder | yes | yes | same | [plugins.md](plugins.md) | | Startup / Shutdown folders | yes | not run | missing | plugins.md | | `#include`, `#target` directives | yes | a script starting with one doesn't parse | missing | plugins.md; common in real scripts |