From 96a63855a98b4a05c24f9095b949aebf29d8665a Mon Sep 17 00:00:00 2001 From: Elias Date: Sat, 10 Oct 2026 03:14:33 +0100 Subject: [PATCH] AG6: journal replay repros Bug reports become replay scripts and crash regression tests become data files: - engine::replay: a JSON-lines repro format (start line, command steps with expect_error, expect queries with JSON pointer + equals, expect_no_error), a never-panicking runner that reports per-step ok/error and the first failure, and journal export (from the last project open; saves left out; size-capped). - journal.export {path?} and help.exportRepro {path?} (not journaled). - Crash capture: when the last-resort guard catches a panic in the outermost command, the journal plus that command is saved to Repros/crash-*.jsonl in the settings store (newest 20 kept) and the error names the file. - Commands run by a journaled command (script steps) are no longer journaled twice. - effectcraft-cli replay SCRIPT.jsonl [--json] [--keep-going]: exits 1 on a failed step, unmet expectation or panic. - crates/host/tests/repros.rs replays every tests/repros/*.jsonl; three samples from past crash fixes (zero frame rate, Layer Settings with no selection, New Comp from Selection with hostile overlap). - Docs: docs/agents.md "Repro scripts"; AGENTS.md: crash fix regression test = a .jsonl in tests/repros. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit c38509d7dafe507856c48c78db65e00b94b0cebc) (cherry picked from commit e55b56dbf76716adfcbfe4cc773df19a74edc419) --- AGENTS.md | 2 + apps/effectcraft-cli/src/main.rs | 58 +- apps/effectcraft-cli/tests/cli.rs | 28 + crates/engine/src/commands/journal.rs | 53 ++ crates/engine/src/commands/mod.rs | 2 + crates/engine/src/guard.rs | 14 + crates/engine/src/lib.rs | 12 +- crates/engine/src/replay.rs | 693 ++++++++++++++++++ crates/host/tests/repros.rs | 31 + docs/agents.md | 62 ++ tests/repros/comp-zero-frame-rate.jsonl | 13 + .../layer-settings-empty-selection.jsonl | 8 + ...-comp-from-selection-hostile-overlap.jsonl | 17 + 13 files changed, 989 insertions(+), 4 deletions(-) create mode 100644 crates/engine/src/commands/journal.rs create mode 100644 crates/engine/src/replay.rs create mode 100644 crates/host/tests/repros.rs create mode 100644 tests/repros/comp-zero-frame-rate.jsonl create mode 100644 tests/repros/layer-settings-empty-selection.jsonl create mode 100644 tests/repros/new-comp-from-selection-hostile-overlap.jsonl diff --git a/AGENTS.md b/AGENTS.md index 464350f32..e16cec1a4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,8 @@ building on top of it. Full standard: [craftrules `standards/never-crash.md`](ht `catch_unwind` boundary that turns an escaped panic into an error and keeps the document. It is a safety net, not a substitute for the rules above; keep `panic = "unwind"`. - **Prove it:** every crash fix comes with a small synthetic regression test that panicked before the fix. + Crash fix regression test = drop a `.jsonl` in `tests/repros` (the crash repro, trimmed; see + [docs/agents.md](docs/agents.md), "Repro scripts"). Tests, benches, fuzz targets and `xtask` may `unwrap()`; `clippy.toml` allows it in tests. Crates that are clean carry `#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, diff --git a/apps/effectcraft-cli/src/main.rs b/apps/effectcraft-cli/src/main.rs index fe5238075..1295846ed 100644 --- a/apps/effectcraft-cli/src/main.rs +++ b/apps/effectcraft-cli/src/main.rs @@ -24,6 +24,7 @@ //! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N] everyday-operation timings //! on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel) //! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE run an After Effects-style script +//! effectcraft-cli replay SCRIPT.jsonl [--keep-going] replay a repro script (docs/agents.md) //! effectcraft-cli mcp [--autosave | --bridge PORT] MCP server on stdio //! //! Project: --project F.ecproj (or a positional *.ecproj) | --demo | --empty (default: demo; mcp: empty) @@ -89,7 +90,7 @@ fn stdout_broken() -> bool { STDOUT_FAILED.get().is_some_and(|k| *k != std::io::ErrorKind::BrokenPipe) } -const USAGE: &str = "usage: effectcraft-cli [args] [--json] +const USAGE: &str = "usage: effectcraft-cli [args] [--json] info project + engine summary commands [--filter TEXT] [--enabled] list engine commands exec [--params JSON] run one engine command (exec --list: list them, as `commands`) @@ -116,6 +117,9 @@ const USAGE: &str = "usage: effectcraft-cli Result<(), Failure> { Ok(()) } +/// `replay SCRIPT.jsonl`: run a repro script against a headless session (its start line picks +/// the project) and report every step. Exits 1 when the replay fails. +fn replay_cmd(args: &Args, json_out: bool) -> Result<(), Failure> { + use effectcraft_engine::replay; + let Some(file) = args.pos.first() else { return usage_err("replay needs a .jsonl script") }; + if args.opt("--bridge").is_some() || args.project.is_some() || args.flag("--demo") || args.flag("--empty") { + return usage_err("replay runs headless from the script's start line ({\"start\": \"new\" | \"demo\" | {\"project\": path}})"); + } + let path = std::path::Path::new(file); + let size = std::fs::metadata(path).map_err(|e| Failure::Error(format!("cannot read {file}: {e}")))?.len(); + if size > replay::MAX_SCRIPT_BYTES as u64 { + return Err(Failure::Error(format!("{file} is larger than {} MB", replay::MAX_SCRIPT_BYTES >> 20))); + } + let script = std::fs::read_to_string(path).map_err(|e| Failure::Error(format!("cannot read {file}: {e}")))?; + let mut s = session(args)?; + let opts = replay::Options { keep_going: args.flag("--keep-going"), base_dir: path.parent().map(std::path::Path::to_path_buf) }; + let report = replay::run(&mut s, &script, &opts); + if json_out { + emit(&serde_json::to_value(&report).unwrap_or_else(|e| json!({"error": e.to_string()})), true); + } else { + for st in &report.steps { + let what = st.command.as_deref().unwrap_or(st.kind); + match (&st.error, st.ok) { + (None, _) => say!("line {:>4} ok {what}", st.line), + (Some(e), true) => say!("line {:>4} ok {what} (tolerated error: {e})", st.line), + (Some(e), false) => say!("line {:>4} FAIL {what}: {e}", st.line), + } + } + } + let Some(first) = &report.first_failure else { + if !json_out { + say!("replay ok: {} steps", report.steps.len()); + } + return Ok(()); + }; + let msg = format!( + "failed at line {} ({}){}: {}{}", + first.line, + first.command.as_deref().unwrap_or(first.kind), + if report.panicked { ", a command panicked" } else { "" }, + first.error.as_deref().unwrap_or(""), + if report.skipped > 0 { format!(" ({} later steps not run)", report.skipped) } else { String::new() } + ); + if json_out { + note!("effectcraft-cli replay: {msg}"); + std::process::exit(1); + } + Err(Failure::Error(msg)) +} + /// Print a result: compact JSON with `--json`, else pretty JSON. fn emit(v: &Value, json_out: bool) { if json_out { @@ -485,6 +540,7 @@ fn run(cmd: &str, args: &Args, json_out: bool) -> Result<(), Failure> { "commands" => list_commands_cmd(args, json_out)?, "exec" if args.flag("--list") => list_commands_cmd(args, json_out)?, "script" => script_cmd(args, json_out)?, + "replay" => replay_cmd(args, json_out)?, "exec" => { let Some(id) = args.pos.first().cloned() else { return usage_err("exec needs a command id") }; let params = match (args.opt("--params"), args.pos.get(1).map(String::as_str)) { diff --git a/apps/effectcraft-cli/tests/cli.rs b/apps/effectcraft-cli/tests/cli.rs index 62aefd5d6..2f8e4e373 100644 --- a/apps/effectcraft-cli/tests/cli.rs +++ b/apps/effectcraft-cli/tests/cli.rs @@ -491,3 +491,31 @@ fn render_bitrate_applies_to_vp9_webm() { assert!(high > low, "100 kbps: {low} bytes, 8000 kbps: {high} bytes"); let _ = std::fs::remove_dir_all(&root); } + +/// `replay`: a repro script runs headless with per-step results; a failing step exits 1 and is +/// named as the first failure; malformed lines are failures, not crashes. +#[test] +fn replay_reports_steps_and_the_first_failure() { + let repro = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/repros/comp-zero-frame-rate.jsonl"); + let v = ok_json(&["replay", repro.to_str().unwrap()]); + assert_eq!(v["ok"], true, "{v}"); + assert!(v["steps"].as_array().unwrap().len() > 5); + let bad = tmp("bad-repro.jsonl"); + std::fs::write( + &bad, + "{\"start\": \"new\"}\n{\"command\": \"comp.new\"}\n{\"expect\": {\"query\": \"comp.info\", \"pointer\": \"/width\", \"equals\": 1}}\nnot json\n", + ) + .unwrap(); + let (code, v) = run_json(&["replay", bad.to_str().unwrap()]); + assert_eq!(code, 1); + assert_eq!(v["ok"], false); + assert_eq!(v["first_failure"]["line"], 3, "{v}"); + assert_eq!(v["skipped"], 1); + let out = bin().args(["replay", bad.to_str().unwrap(), "--keep-going"]).output().unwrap(); + assert_eq!(out.status.code(), Some(1)); + let text = String::from_utf8_lossy(&out.stdout); + assert!(text.contains("line 2 ok comp.new") && text.contains("line 4 FAIL invalid: not JSON"), "{text}"); + assert!(String::from_utf8_lossy(&out.stderr).contains("failed at line 3 (comp.info)")); + // The start line picks the project. + assert_eq!(bin().args(["replay", bad.to_str().unwrap(), "--demo"]).output().unwrap().status.code(), Some(2)); +} diff --git a/crates/engine/src/commands/journal.rs b/crates/engine/src/commands/journal.rs new file mode 100644 index 000000000..4b97a5ee1 --- /dev/null +++ b/crates/engine/src/commands/journal.rs @@ -0,0 +1,53 @@ +//! The command journal as a replay script (see [`crate::replay`]): `journal.export` for agents +//! and `help.exportRepro` for bug reports. Neither is journaled. + +use serde_json::{Value, json}; + +use super::{CommandSpec, always, str_p}; +use crate::{Result, Session, replay}; + +/// `journal.export {path?}`: the script itself, or written to `path`. +fn export(s: &mut Session, p: &Value) -> Result { + if let Some(path) = str_p(p, "path") { + let (path, steps) = replay::save_repro(s, Some(path))?; + return Ok(json!({"path": path, "steps": steps})); + } + let script = replay::export(s, None); + Ok(json!({"steps": script.lines().count().saturating_sub(1), "script": script})) +} + +/// `help.exportRepro {path?}`: save the steps so far for a bug report (default: the settings +/// folder's `Repros`). +fn export_repro(s: &mut Session, p: &Value) -> Result { + let (path, steps) = replay::save_repro(s, str_p(p, "path"))?; + Ok(json!({"path": path, "steps": steps})) +} + +const fn spec(id: &'static str, label: &'static str, params: &'static str, run: super::Run) -> CommandSpec { + CommandSpec { id, label, menu: &[], shortcut: None, params, enabled: always, run, journal: false } +} + +pub fn specs() -> Vec { + #[allow(unused_mut)] + let mut v = vec![ + spec( + "journal.export", + "Export Command Journal", + "{path? (write the script there)} → {script, steps} or {path, steps}: the commands run since the project was last opened, as a replay script (docs/agents.md, Repro scripts)", + export, + ), + spec( + "help.exportRepro", + "Save Bug Repro...", + "{path? (default: Repros in the settings folder)} → {path, steps}: the steps since the project was opened, for a bug report (commands and parameters only)", + export_repro, + ), + ]; + // A command with a bug, for the crash-repro tests. + #[cfg(test)] + v.push(spec("test.panic", "Panic (test)", "{why?}", |_, p| { + let why = str_p(p, "why").unwrap_or("test").to_string(); + std::panic::panic_any(why) + })); + v +} diff --git a/crates/engine/src/commands/mod.rs b/crates/engine/src/commands/mod.rs index 98fe0cb49..a7b215494 100644 --- a/crates/engine/src/commands/mod.rs +++ b/crates/engine/src/commands/mod.rs @@ -25,6 +25,7 @@ pub mod footage_panel; pub(crate) mod frame_export; mod frontend; mod help; +mod journal; mod key_labels; mod key_transform; mod keys_more; @@ -157,6 +158,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(time::specs()); v.extend(render_queue::specs()); v.extend(help::specs()); + v.extend(journal::specs()); v.extend(query::specs()); v.extend(batch::specs()); v.extend(layer_menu::specs()); diff --git a/crates/engine/src/guard.rs b/crates/engine/src/guard.rs index 699e6474d..ec32e2545 100644 --- a/crates/engine/src/guard.rs +++ b/crates/engine/src/guard.rs @@ -4,14 +4,28 @@ //! panics leaves the project as it was. This is a safety net, not a substitute for returning //! errors (AGENTS.md, "Never crash"). +use std::cell::Cell; + use crate::{EngineError, Result}; +thread_local! { + /// Panics [`guarded`] caught on this thread (see [`caught`]). + static CAUGHT: Cell = const { Cell::new(0) }; +} + +/// How many panics [`guarded`] has caught on this thread so far: compare two readings to learn +/// whether a call panicked (crash repros, `effectcraft-cli replay`). +pub fn caught() -> u64 { + CAUGHT.with(Cell::get) +} + /// Run `f`; a panic inside it becomes an error naming `what`. The panic itself is logged by /// the panic hook ([`crate::logging::install_panic_hook`]). pub fn guarded(what: &str, f: impl FnOnce() -> Result) -> Result { match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) { Ok(r) => r, Err(payload) => { + CAUGHT.with(|c| c.set(c.get().saturating_add(1))); let msg = payload .downcast_ref::<&str>() .map(|s| s.to_string()) diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index a9e9b606b..1e240f44e 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -35,6 +35,7 @@ pub mod preview; pub mod psd_import; pub mod remote; pub mod render_queue; +pub mod replay; pub mod roto; pub mod scriptui; pub mod sequence; @@ -598,10 +599,15 @@ impl Session { return Err(EngineError::Disabled(id.to_string(), why)); } } - // Last-resort guard: a panicking command reports an error instead of crashing the app. - let r = guard::guarded(&format!("command `{id}`"), || (spec.run)(self, ¶ms))?; + // Last-resort guard: a panicking command reports an error instead of crashing the app + // (and saves a repro of the steps that led there, see `replay::after_command`). + let nesting = replay::Nesting::enter(spec.journal); + let caught = guard::caught(); + let r = guard::guarded(&format!("command `{id}`"), || (spec.run)(self, ¶ms)); + let r = replay::after_command(self, id, ¶ms, &nesting, caught, r)?; self.learn_observe(id, ¶ms); - if spec.journal { + // Commands run by a journaled command (a script's steps) replay with it, not again. + if spec.journal && !nesting.inside_journaled() { self.journal.push((id.to_string(), params)); if self.journal.len() > 10_000 { self.journal.drain(..1000); diff --git a/crates/engine/src/replay.rs b/crates/engine/src/replay.rs new file mode 100644 index 000000000..b8e027d61 --- /dev/null +++ b/crates/engine/src/replay.rs @@ -0,0 +1,693 @@ +//! Replay scripts: a bug or crash as a list of commands an agent (or a test) can run again. +//! +//! The session's command journal ([`Session::journal`]) becomes a JSON-lines script +//! (`journal.export`, `help.exportRepro`, and automatically when the last-resort guard catches a +//! panic, see [`after_command`]); `effectcraft-cli replay script.jsonl` runs one headless and +//! `crates/host/tests/repros.rs` runs every `tests/repros/*.jsonl` as a regression test. +//! +//! Format (one JSON object per line; blank lines and lines starting with `//` or `#` are +//! comments): +//! +//! ```text +//! {"start": "new" | "demo" | {"project": "path.ecproj"}, "allow_errors"?: bool} optional, first +//! {"command": "comp.new", "params": {...}, "expect_error"?: true | "substring"} +//! {"expect": {"query": "prop.get", "params": {...}, "pointer"?: "/value", "equals"?: ...}} +//! {"expect_no_error": true} +//! ``` +//! +//! A command must succeed unless it says `expect_error` (then it must fail, with that text in +//! the message when it is a string), or the start line says `allow_errors` (errors are reported +//! but tolerated; crash repros say so, since after the fix the crashing step may fail cleanly). +//! A panic always fails. `expect` runs a command (normally a query) and compares the value at +//! the JSON `pointer` of its result with `equals` (numbers within 1e-6); `expect_no_error` +//! checks that the previous command succeeded. Other keys (`version`, `app`, `note`, `crash`…) +//! are information for people. Relative `project` paths are relative to the script. +//! +//! Privacy: a repro holds command ids and their parameters only, never project or media file +//! contents. Parameters can include file paths, layer names and text typed into text layers. + +use std::cell::Cell; +use std::path::{Path, PathBuf}; + +use serde::Serialize; +use serde_json::{Map, Value, json}; + +use crate::{EngineError, Result, Session, guard}; + +/// The format version written into exported headers. +pub const VERSION: u64 = 1; +/// Largest script [`run`] accepts (bytes). +pub const MAX_SCRIPT_BYTES: usize = 64 << 20; +/// Most lines [`run`] reads. +pub const MAX_LINES: usize = 100_000; +/// Exported scripts stay below this size: the oldest steps are dropped (and counted). +const MAX_EXPORT_BYTES: usize = 32 << 20; +/// Folder of saved repros in the settings store (next to `Logs`). +pub const REPRO_DIR: &str = "Repros"; +/// Automatic crash repros kept (the oldest are removed). +const KEEP_CRASH_REPROS: usize = 20; +/// Commands that replace the whole project: an export starts at the last one. +const PROJECT_STARTS: &[&str] = &["file.newProject", "file.open", "file.openDemoProject"]; +/// Error and value texts in reports are cut to this many characters. +const MAX_TEXT: usize = 600; + +// ---------- nesting and crash capture (Session::execute) ---------- + +thread_local! { + /// Commands running on this thread: (all, journaled ones). + static DEPTH: Cell<(u32, u32)> = const { Cell::new((0, 0)) }; + /// A replay is running on this thread (its panics are reported, not saved as repros). + static REPLAYING: Cell = const { Cell::new(false) }; +} + +/// One running command (see [`Session::execute`]): commands it runs itself (a script's steps) +/// aren't journaled again, and only the outermost command saves a crash repro. +pub struct Nesting { + outer: (u32, u32), +} + +impl Nesting { + pub fn enter(journaled: bool) -> Nesting { + let outer = DEPTH.with(Cell::get); + DEPTH.with(|d| d.set((outer.0.saturating_add(1), outer.1.saturating_add(u32::from(journaled))))); + Nesting { outer } + } + /// Run by a journaled command (which replays it). + pub fn inside_journaled(&self) -> bool { + self.outer.1 > 0 + } + pub fn outermost(&self) -> bool { + self.outer.0 == 0 + } +} + +impl Drop for Nesting { + fn drop(&mut self) { + DEPTH.with(|d| d.set(self.outer)); + } +} + +/// After the outermost command: when the guard caught a panic while it ran (`caught` is +/// [`guard::caught`] from before), save the journal plus this command as a crash repro in the +/// settings store and name it in the error. +pub fn after_command(s: &Session, id: &str, params: &Value, nesting: &Nesting, caught: u64, r: Result) -> Result { + if !nesting.outermost() || guard::caught() == caught || REPLAYING.with(Cell::get) { + return r; + } + let error = match &r { + Err(e) => e.to_string(), + Ok(_) => "a step it ran failed with an internal error".to_string(), + }; + let saved = save_crash_repro(s, id, params, &error); + match (r, saved) { + (Err(EngineError::Other(m)), Some(path)) => Err(EngineError::Other(format!("{m} (the steps that led here were saved to {path})"))), + (r, _) => r, + } +} + +fn unix_secs() -> u64 { + web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0) +} + +/// `Repros/--.jsonl`. +fn repro_name(prefix: &str, id: &str) -> String { + let id: String = id.chars().map(|c| if c.is_ascii_alphanumeric() || c == '.' { c } else { '_' }).take(60).collect(); + format!("{REPRO_DIR}/{prefix}-{:012}-{id}.jsonl", unix_secs()) +} + +/// Where a store entry is on disk (or its name in stores without a folder). +fn store_path(store: &dyn crate::config::ConfigStore, name: &str) -> String { + store.dir().map(|d| d.join(name).display().to_string()).unwrap_or_else(|| name.to_string()) +} + +fn save_crash_repro(s: &Session, id: &str, params: &Value, error: &str) -> Option { + let store = s.config.clone()?; + let mut text = export(s, Some(json!({"command": id, "error": cut(error)}))); + text.push_str(&command_line(id, params)); + text.push('\n'); + let name = repro_name("crash", id); + if let Err(e) = store.write(&name, &text) { + log::error!("cannot save the crash repro {name}: {e}"); + return None; + } + // Keep the newest few (names sort by time). + let mut old: Vec = store.list(REPRO_DIR).into_iter().filter(|n| n.starts_with("crash-")).collect(); + old.sort(); + let extra = old.len().saturating_sub(KEEP_CRASH_REPROS); + for n in old.into_iter().take(extra) { + let _ = store.remove(&format!("{REPRO_DIR}/{n}")); + } + let path = store_path(store.as_ref(), &name); + log::error!("crash repro saved to {path}"); + Some(path) +} + +/// Save the journal as a repro: at `path` (through the session's file services), else in the +/// settings store's `Repros` folder. Returns where it went and how many steps it holds. +pub fn save_repro(s: &Session, path: Option<&str>) -> Result<(String, usize)> { + let text = export(s, None); + let steps = text.lines().count().saturating_sub(1); + if let Some(path) = path { + s.services.write_file(path, text.as_bytes()).map_err(|e| EngineError::Other(format!("cannot write {path}: {e}")))?; + return Ok((path.to_string(), steps)); + } + let store = s.config.clone().ok_or_else(|| EngineError::Other("there is no settings folder to save the repro in; pass `path`".into()))?; + let name = repro_name("repro", "session"); + store.write(&name, &text).map_err(|e| EngineError::Other(format!("cannot save {name}: {e}")))?; + Ok((store_path(store.as_ref(), &name), steps)) +} + +// ---------- export ---------- + +fn command_line(id: &str, params: &Value) -> String { + let mut o = Map::new(); + o.insert("command".into(), json!(id)); + if !(params.is_null() || params.as_object().is_some_and(Map::is_empty)) { + o.insert("params".into(), params.clone()); + } + Value::Object(o).to_string() +} + +/// Saving writes the user's files and doesn't change the project: left out of repros. +fn skipped_in_export(id: &str) -> bool { + id.starts_with("file.save") || id.starts_with("file.increment") +} + +/// The journal as a replay script: a start line, then the steps since the project was last +/// replaced (`file.newProject` / `file.open` / `file.openDemoProject`). `crash` (the failing +/// command and its error) marks a crash repro, which tolerates errors on replay. +pub fn export(s: &Session, crash: Option) -> String { + let from = s.journal.iter().rposition(|(id, _)| PROJECT_STARTS.contains(&id.as_str())).unwrap_or(0); + let mut lines: Vec = + s.journal.get(from..).unwrap_or_default().iter().filter(|(id, _)| !skipped_in_export(id)).map(|(id, p)| command_line(id, p)).collect(); + let mut size: usize = lines.iter().map(|l| l.len().saturating_add(1)).sum(); + let mut dropped = 0usize; + while size > MAX_EXPORT_BYTES && !lines.is_empty() { + size = size.saturating_sub(lines.remove(0).len().saturating_add(1)); + dropped += 1; + } + let mut head = Map::new(); + head.insert("start".into(), json!("new")); + head.insert("version".into(), json!(VERSION)); + head.insert("app".into(), json!(format!("EffectCraft {}", env!("CARGO_PKG_VERSION")))); + head.insert("os".into(), json!(std::env::consts::OS)); + head.insert("exported".into(), json!(unix_secs())); + if let Some(c) = crash { + head.insert("allow_errors".into(), json!(true)); + head.insert("crash".into(), c); + } + if dropped > 0 { + head.insert("dropped".into(), json!(dropped)); + } + head.insert("note".into(), json!("commands and parameters only (no file contents); parameters can include file paths and text")); + let mut out = Value::Object(head).to_string(); + out.push('\n'); + for l in lines { + out.push_str(&l); + out.push('\n'); + } + out +} + +// ---------- parsing ---------- + +#[derive(Clone, Debug, PartialEq)] +enum Start { + New, + Demo, + Project(String), +} + +#[derive(Clone, Debug)] +enum Step { + Start(Start), + Command { + id: String, + params: Value, + expect_error: Option>, + }, + Expect { + query: String, + params: Value, + pointer: Option, + equals: Option, + }, + ExpectNoError, + /// A line that isn't a valid step (reported as a failure). + Invalid(String), +} + +fn params_of(v: Option<&Value>) -> std::result::Result { + match v { + None | Some(Value::Null) => Ok(json!({})), + Some(p @ Value::Object(_)) => Ok(p.clone()), + Some(_) => Err("`params` must be an object".into()), + } +} + +fn parse_start(v: &Value) -> std::result::Result { + match v { + Value::String(s) if s == "new" => Ok(Start::New), + Value::String(s) if s == "demo" => Ok(Start::Demo), + Value::Object(o) => match o.get("project").and_then(Value::as_str) { + Some(p) if !p.trim().is_empty() => Ok(Start::Project(p.to_string())), + _ => Err("`start` object needs a `project` path".into()), + }, + _ => Err("`start` must be \"new\", \"demo\" or {\"project\": path}".into()), + } +} + +/// One line → a step (`None`: a blank or comment line). +fn parse_line(text: &str, first: bool, allow_errors: &mut bool) -> Option { + let t = text.trim(); + if t.is_empty() || t.starts_with("//") || t.starts_with('#') { + return None; + } + let v: Value = match serde_json::from_str(t) { + Ok(v) => v, + Err(e) => return Some(Step::Invalid(format!("not JSON: {e}"))), + }; + let Some(o) = v.as_object() else { return Some(Step::Invalid("a line must be a JSON object".into())) }; + let step = if let Some(st) = o.get("start") { + if !first { + return Some(Step::Invalid("the `start` line must come first".into())); + } + *allow_errors = o.get("allow_errors").and_then(Value::as_bool).unwrap_or(false); + parse_start(st).map(Step::Start) + } else if let Some(c) = o.get("command") { + let Some(id) = c.as_str().filter(|s| !s.is_empty()) else { return Some(Step::Invalid("`command` must be a command id".into())) }; + let expect_error = match o.get("expect_error") { + None | Some(Value::Null | Value::Bool(false)) => Ok(None), + Some(Value::Bool(true)) => Ok(Some(None)), + Some(Value::String(s)) => Ok(Some(Some(s.clone()))), + Some(_) => Err("`expect_error` must be true or a text the error contains".to_string()), + }; + match (params_of(o.get("params")), expect_error) { + (Ok(params), Ok(expect_error)) => Ok(Step::Command { id: id.to_string(), params, expect_error }), + (Err(m), _) | (_, Err(m)) => Err(m), + } + } else if let Some(e) = o.get("expect") { + let query = e.get("query").and_then(Value::as_str).filter(|s| !s.is_empty()); + let pointer = e.get("pointer").map(|p| p.as_str().map(str::to_string).ok_or("`pointer` must be a JSON pointer such as /value")); + match (query, pointer.transpose()) { + (None, _) => Err("`expect` needs a `query` command id".to_string()), + (_, Err(m)) => Err(m.to_string()), + (Some(q), Ok(pointer)) => { + params_of(e.get("params")).map(|params| Step::Expect { query: q.to_string(), params, pointer, equals: e.get("equals").cloned() }) + } + } + } else if o.get("expect_no_error").and_then(Value::as_bool) == Some(true) { + Ok(Step::ExpectNoError) + } else if o.len() == 1 && o.contains_key("comment") { + return None; + } else { + Err("expected a `start`, `command`, `expect` or `expect_no_error` line".to_string()) + }; + Some(step.unwrap_or_else(Step::Invalid)) +} + +// ---------- running ---------- + +/// How to run a script. +#[derive(Clone, Debug, Default)] +pub struct Options { + /// Keep running after a failed step (default: stop at the first failure). + pub keep_going: bool, + /// Folder relative `start.project` paths are resolved against (the script's). + pub base_dir: Option, +} + +/// What happened to one line. +#[derive(Clone, Debug, Serialize)] +pub struct StepReport { + /// 1-based line number (0: the implicit start). + pub line: usize, + /// `start`, `command`, `expect`, `expect_no_error` or `invalid`. + pub kind: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub command: Option, + pub ok: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub panicked: bool, +} + +/// The outcome of [`run`]. +#[derive(Clone, Debug, Default, Serialize)] +pub struct Report { + pub ok: bool, + /// A step panicked (caught by the last-resort guard): always a failure. + pub panicked: bool, + pub steps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub first_failure: Option, + /// Steps not run after the first failure. + pub skipped: usize, +} + +/// Cut a message to [`MAX_TEXT`] characters. +fn cut(s: &str) -> String { + match s.char_indices().nth(MAX_TEXT) { + Some((i, _)) => format!("{}…", s.get(..i).unwrap_or(s)), + None => s.to_string(), + } +} + +/// Run one command the way the app does, noting whether it panicked. +fn exec(s: &mut Session, id: &str, params: Value) -> (Result, bool) { + let before = guard::caught(); + let r = guard::guarded(&format!("replay of `{id}`"), || s.execute(id, params)); + s.events.clear(); + (r, guard::caught() != before) +} + +/// Numbers within 1e-6 (relative above 1), everything else exactly. +fn approx_eq(a: &Value, b: &Value) -> bool { + match (a, b) { + (Value::Number(x), Value::Number(y)) => match (x.as_f64(), y.as_f64()) { + (Some(x), Some(y)) => (x - y).abs() <= 1e-6 * x.abs().max(y.abs()).max(1.0), + _ => x == y, + }, + (Value::Array(x), Value::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(a, b)| approx_eq(a, b)), + (Value::Object(x), Value::Object(y)) => x.len() == y.len() && x.iter().all(|(k, a)| y.get(k).is_some_and(|b| approx_eq(a, b))), + _ => a == b, + } +} + +/// Marks this thread as replaying while alive. +struct Replaying(bool); + +impl Replaying { + fn enter() -> Replaying { + Replaying(REPLAYING.with(|r| r.replace(true))) + } +} + +impl Drop for Replaying { + fn drop(&mut self) { + REPLAYING.with(|r| r.set(self.0)); + } +} + +fn start_step(s: &mut Session, start: &Start, base: Option<&Path>) -> (String, Result, bool) { + match start { + Start::New => { + let (r, p) = exec(s, "file.newProject", json!({})); + ("file.newProject".into(), r, p) + } + Start::Demo => { + let (r, p) = exec(s, "file.openDemoProject", json!({})); + ("file.openDemoProject".into(), r, p) + } + Start::Project(path) => { + let full = match base { + Some(b) if Path::new(path).is_relative() => b.join(path).display().to_string(), + _ => path.clone(), + }; + let (r, p) = exec(s, "file.open", json!({"path": full})); + ("file.open".into(), r, p) + } + } +} + +/// Run a replay script against `s`. Never panics: malformed lines are failed steps. +pub fn run(s: &mut Session, script: &str, opts: &Options) -> Report { + let _replaying = Replaying::enter(); + let mut report = Report { ok: true, ..Default::default() }; + let fail = |report: &mut Report, sr: StepReport| { + report.ok = false; + report.panicked |= sr.panicked; + if report.first_failure.is_none() { + report.first_failure = Some(sr.clone()); + } + report.steps.push(sr); + }; + if script.len() > MAX_SCRIPT_BYTES { + let msg = format!("the script is larger than {} MB", MAX_SCRIPT_BYTES >> 20); + fail(&mut report, StepReport { line: 0, kind: "invalid", command: None, ok: false, error: Some(msg), panicked: false }); + return report; + } + let mut allow_errors = false; + let mut steps: Vec<(usize, Step)> = vec![]; + for (i, line) in script.lines().enumerate().take(MAX_LINES) { + if let Some(step) = parse_line(line, steps.is_empty(), &mut allow_errors) { + steps.push((i + 1, step)); + } + } + if script.lines().nth(MAX_LINES).is_some() { + steps.push((MAX_LINES + 1, Step::Invalid(format!("the script has more than {MAX_LINES} lines; the rest was not read")))); + } + if !matches!(steps.first(), Some((_, Step::Start(_)) | (_, Step::Invalid(_)))) { + steps.insert(0, (0, Step::Start(Start::New))); + } + let mut last_command_failed: Option = None; + let mut stopped = false; + for (line, step) in steps { + if stopped { + report.skipped += 1; + continue; + } + let mut sr = StepReport { line, kind: "command", command: None, ok: true, error: None, panicked: false }; + match step { + Step::Invalid(m) => { + sr.kind = "invalid"; + sr.ok = false; + sr.error = Some(m); + } + Step::Start(start) => { + sr.kind = "start"; + let (id, r, panicked) = start_step(s, &start, opts.base_dir.as_deref()); + sr.command = Some(id); + sr.panicked = panicked; + if let Err(e) = r { + sr.ok = false; + sr.error = Some(cut(&e.to_string())); + } + } + Step::Command { id, params, expect_error } => { + let (r, panicked) = exec(s, &id, params); + sr.command = Some(id); + sr.panicked = panicked; + last_command_failed = Some(r.is_err()); + let err = r.err().map(|e| e.to_string()); + let (ok, error) = match (err, expect_error) { + (e, _) if panicked => (false, Some(format!("panicked: {}", e.unwrap_or_else(|| "a step failed with an internal error".into())))), + (None, None) => (true, None), + (None, Some(_)) => (false, Some("expected an error, but the command succeeded".into())), + (Some(e), Some(Some(want))) if !e.contains(want.as_str()) => (false, Some(format!("expected an error containing {want:?}, got: {e}"))), + (Some(_), Some(_)) => (true, None), + (Some(e), None) => (allow_errors, Some(e)), + }; + sr.ok = ok; + sr.error = error.as_deref().map(cut); + } + Step::Expect { query, params, pointer, equals } => { + sr.kind = "expect"; + let (r, panicked) = exec(s, &query, params); + sr.command = Some(query); + sr.panicked = panicked; + match r { + Err(e) => { + sr.ok = false; + sr.error = Some(cut(&format!("{}{e}", if panicked { "panicked: " } else { "" }))); + } + Ok(v) => { + let got = match &pointer { + Some(p) => v.pointer(p), + None => Some(&v), + }; + match (got, &equals) { + (None, _) => { + sr.ok = false; + sr.error = Some(cut(&format!("no value at {} in {v}", pointer.as_deref().unwrap_or("")))); + } + (Some(g), Some(want)) if !approx_eq(g, want) => { + sr.ok = false; + sr.error = Some(cut(&format!("expected {want}, got {g}"))); + } + _ => sr.ok = !panicked, + } + } + } + } + Step::ExpectNoError => { + sr.kind = "expect_no_error"; + match last_command_failed { + Some(false) => {} + Some(true) => { + sr.ok = false; + sr.error = Some("the previous command failed".into()); + } + None => { + sr.ok = false; + sr.error = Some("no command before this line".into()); + } + } + } + } + if sr.ok { + report.steps.push(sr); + } else { + fail(&mut report, sr); + stopped = !opts.keep_going; + } + } + report +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use serde_json::json; + + use super::{Options, run}; + use crate::Session; + use crate::config::{ConfigStore, MemoryConfig}; + + fn replay(script: &str) -> super::Report { + run(&mut Session::new(), script, &Options::default()) + } + + #[test] + fn steps_expectations_and_failures_are_reported() { + let r = replay( + r#"{"start": "new"} +// a comment +{"command": "comp.new", "params": {"name": "A", "width": 320, "height": 180, "frameRate": 30, "duration": 2}} +{"expect_no_error": true} +{"command": "comp.new", "params": {"frameRate": 0}, "expect_error": "frame"} +{"expect": {"query": "comp.info", "pointer": "/width", "equals": 320}} +"#, + ); + assert!(r.ok, "{r:?}"); + assert_eq!(r.steps.len(), 5); + let r = replay( + "{\"command\": \"comp.new\"}\n{\"expect\": {\"query\": \"comp.info\", \"pointer\": \"/width\", \"equals\": 1}}\n{\"command\": \"comp.new\"}", + ); + assert!(!r.ok && !r.panicked && r.skipped == 1, "{r:?}"); + assert_eq!(r.first_failure.as_ref().map(|f| f.line), Some(2)); + // An unexpected success, an unknown command. + assert!(!replay(r#"{"command": "comp.new", "expect_error": true}"#).ok); + assert!(!replay(r#"{"command": "no.such.command"}"#).ok); + // Tolerated errors. + let r = replay("{\"start\": \"new\", \"allow_errors\": true}\n{\"command\": \"no.such.command\"}\n{\"expect_no_error\": true}"); + assert!(!r.ok && r.first_failure.as_ref().map(|f| f.kind) == Some("expect_no_error"), "{r:?}"); + } + + #[test] + fn malformed_lines_are_failures_not_crashes() { + for bad in [ + "not json", + "[1, 2]", + "{\"command\": 5}", + "{\"command\": \"comp.new\", \"params\": [1]}", + "{\"command\": \"comp.new\", \"expect_error\": 3}", + "{\"expect\": {}}", + "{\"expect\": {\"query\": \"comp.info\", \"pointer\": 4}}", + "{\"expect\": {\"query\": \"comp.info\", \"pointer\": \"/nope\"}}", + "{\"start\": 7}", + "{\"start\": {\"project\": \"\"}}", + "{\"command\": \"comp.new\"}\n{\"start\": \"new\"}", + "{\"what\": 1}", + "{\"expect_no_error\": true}", + "{\"start\": {\"project\": \"/no/such/file.ecproj\"}}", + ] { + let r = replay(bad); + assert!(!r.ok && !r.panicked, "{bad}: {r:?}"); + } + let r = replay(&format!("{{\"command\": \"{}\"}}", "\u{1F600}".repeat(1000))); + assert!(!r.ok && r.first_failure.and_then(|f| f.error).is_some_and(|e| e.ends_with('…'))); + assert!(replay("").ok, "an empty script is a new project"); + } + + #[test] + fn the_journal_exports_and_replays() { + let mut s = Session::new(); + s.execute("comp.new", json!({"name": "Old"})).unwrap(); + s.execute("file.openDemoProject", json!({})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#ff0000", "name": "Red"})).unwrap(); + s.execute("layer.newSolid", json!({"color": "#00ff00", "name": "Green"})).unwrap(); + // A query isn't journaled; neither is saving. + s.execute("comp.info", json!({})).unwrap(); + let text = super::export(&s, None); + let lines: Vec<&str> = text.lines().collect(); + assert_eq!(lines.len(), 4, "{text}"); + assert!(lines[0].contains("\"start\":\"new\"") && lines[1].contains("file.openDemoProject"), "starts at the last project open: {text}"); + let mut again = Session::new(); + let r = run(&mut again, &text, &Options::default()); + assert!(r.ok, "{r:?}"); + let layers = |s: &Session| s.active_comp().map(|c| c.layers.len()); + assert_eq!(layers(&again), layers(&s)); + // journal.export returns it; with `path` it writes the file. + let v = s.execute("journal.export", json!({})).unwrap(); + assert_eq!(v["steps"], 3); + assert_eq!(v["script"].as_str().unwrap().lines().skip(1).collect::>(), lines[1..]); + let dir = std::env::temp_dir().join(format!("ec-replay-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("j.jsonl"); + s.execute("journal.export", json!({"path": path.to_str().unwrap()})).unwrap(); + assert!(std::fs::read_to_string(&path).unwrap().contains("layer.newSolid")); + let _ = std::fs::remove_dir_all(&dir); + assert_eq!(s.journal.iter().filter(|(id, _)| id.starts_with("journal.")).count(), 0); + } + + #[test] + fn help_export_repro_saves_into_the_settings_store() { + let mut s = Session::new(); + assert!(s.execute("help.exportRepro", json!({})).is_err(), "no settings folder and no path"); + let store = Arc::new(MemoryConfig::default()); + s.config = Some(store.clone()); + s.execute("comp.new", json!({})).unwrap(); + let v = s.execute("help.exportRepro", json!({})).unwrap(); + assert_eq!(v["steps"], 1); + let saved = store.list(super::REPRO_DIR); + assert_eq!(saved.len(), 1, "{saved:?}"); + assert!(store.read(v["path"].as_str().unwrap()).unwrap().contains("comp.new")); + } + + #[test] + fn a_panicking_command_saves_a_crash_repro_and_replays_as_a_failure() { + let mut s = Session::new(); + let store = Arc::new(MemoryConfig::default()); + s.config = Some(store.clone()); + s.execute("comp.new", json!({"name": "C"})).unwrap(); + // `test.panic` stands in for a command with a bug (see `commands::journal`). + let e = s.execute("test.panic", json!({"why": "boom"})).unwrap_err().to_string(); + assert!(e.contains("boom") && e.contains("Repros/crash-"), "{e}"); + let names = store.list(super::REPRO_DIR); + assert_eq!(names.len(), 1); + let text = store.read(&format!("{}/{}", super::REPRO_DIR, names[0])).unwrap(); + assert!(text.lines().next().unwrap().contains("\"crash\""), "{text}"); + assert!(text.lines().last().unwrap().contains("test.panic"), "{text}"); + // The replay fails on the panic (and doesn't save another repro). + let r = run(&mut Session::new(), &text, &Options::default()); + assert!(!r.ok && r.panicked, "{r:?}"); + assert_eq!(r.first_failure.as_ref().and_then(|f| f.command.as_deref()), Some("test.panic")); + // A failing replay inside a session with a store saves nothing more. + let mut t = Session::new(); + t.config = Some(store.clone()); + assert!(!run(&mut t, &text, &Options::default()).ok); + assert_eq!(store.list(super::REPRO_DIR).len(), 1); + // Only the newest crash repros are kept. + for i in 0..30 { + store.write(&format!("{}/crash-{i:012}-x.jsonl", super::REPRO_DIR), "{}").unwrap(); + } + let _ = s.execute("test.panic", json!({})); + assert_eq!(store.list(super::REPRO_DIR).len(), super::KEEP_CRASH_REPROS); + } + + #[test] + fn commands_run_by_a_journaled_command_are_not_journaled_twice() { + let mut s = Session::new(); + s.execute("comp.new", json!({})).unwrap(); + // engine.batch isn't journaled: its steps are. + s.execute("engine.batch", json!({"steps": [{"command": "layer.newNull"}, {"command": "layer.newNull"}]})).unwrap(); + assert_eq!(s.journal.iter().filter(|(id, _)| id == "layer.newNull").count(), 2); + // A journaled command's own steps replay with it. + s.execute("file.runScript", json!({"steps": [{"command": "layer.newNull"}]})).unwrap(); + let ids: Vec<&str> = s.journal.iter().map(|(id, _)| id.as_str()).collect(); + assert_eq!(ids, ["comp.new", "layer.newNull", "layer.newNull", "file.runScript"]); + } +} diff --git a/crates/host/tests/repros.rs b/crates/host/tests/repros.rs new file mode 100644 index 000000000..1b343603c --- /dev/null +++ b/crates/host/tests/repros.rs @@ -0,0 +1,31 @@ +//! Regression tests as data: every `tests/repros/*.jsonl` (repository root) is a replay script +//! (`effectcraft_engine::replay`, docs/agents.md "Repro scripts") run against a fully wired +//! headless session, like `effectcraft-cli replay`. A crash fix's regression test is a `.jsonl` +//! dropped there: it must replay without a panic and meet its expectations. + +use std::path::{Path, PathBuf}; + +use effectcraft_engine::replay; + +fn repros_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/repros") +} + +#[test] +fn every_repro_replays_cleanly() { + let mut files: Vec = + std::fs::read_dir(repros_dir()).unwrap().map(|e| e.unwrap().path()).filter(|p| p.extension().is_some_and(|x| x == "jsonl")).collect(); + files.sort(); + assert!(!files.is_empty(), "no repros in {}", repros_dir().display()); + let mut failures = vec![]; + for f in &files { + let script = std::fs::read_to_string(f).unwrap(); + let mut s = effectcraft_host::session(); + let opts = replay::Options { keep_going: false, base_dir: f.parent().map(Path::to_path_buf) }; + let r = replay::run(&mut s, &script, &opts); + if !r.ok { + failures.push(format!("{}: {}", f.display(), serde_json::to_string(&r.first_failure).unwrap())); + } + } + assert!(failures.is_empty(), "{} of {} repros failed:\n{}", failures.len(), files.len(), failures.join("\n")); +} diff --git a/docs/agents.md b/docs/agents.md index 6a3b9e85e..c496ca636 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -646,6 +646,68 @@ not read or written. `` is an id or name, or `-` for the active comp. `` is JSON (`50`, `[960,540]`, `"#ff0000"`) or a bare string. +## Repro scripts + +Every command the app runs (menus, panels, shortcuts, agents, scripts) goes into the session's +command journal. A **repro script** is that journal as JSON lines, so a bug or crash becomes +something an agent can replay and a regression test becomes a data file. + +```text +// comments: lines starting with // or #; blank lines are skipped +{"start": "new" | "demo" | {"project": "rel/or/abs.ecproj"}, "allow_errors": false} optional, first line +{"command": "comp.new", "params": {"name": "Main", "frameRate": 30}} +{"command": "comp.settings", "params": {"frameRate": 0}, "expect_error": "frameRate"} +{"expect": {"query": "comp.info", "pointer": "/frameRate", "equals": 30}} +{"expect_no_error": true} +``` + +- `start`: the project to begin with (default `new`). A relative `project` path is relative to the + script. Other header keys (`version`, `app`, `os`, `crash`, `note`) are for people. +- `command`: run like the app runs it. It must succeed, unless it has `expect_error` (`true`, or a + text the error must contain) or the header says `allow_errors` (errors are reported but + tolerated). A panic always fails the replay. +- `expect`: run a command, normally a query (`prop.get`, `comp.info`, `layer.tree`, `project.summary`), + and compare the value at the JSON `pointer` of its result with `equals` (numbers within 1e-6). + Without `equals` it only checks that the query succeeds. +- `expect_no_error`: the previous command succeeded. + +Where scripts come from: + +- `journal.export {path?}` returns `{script, steps}` (or writes `path`): the commands since the + project was last replaced (`file.newProject`, `file.open`, `file.openDemoProject`). Queries and + saves are not in it; commands a script or `file.runScript` ran are covered by that one step. +- `help.exportRepro {path?}` saves the same for a bug report, by default to `Repros/` in the + settings folder (macOS `~/Library/Application Support/EffectCraft/Repros`, next to `Logs/`). +- **Crash repros**: when the last-resort guard catches a panic in a command, the app writes + `Repros/crash--.jsonl` there automatically (the newest 20 are kept): the + journal, then the command that panicked, with `"allow_errors": true` and the error in the header. + The error the user or agent sees names the file. Headless sessions without a settings folder + (the CLI, tests) don't write one. + +Privacy: a repro holds command ids and parameters only, never project, image or media file +contents. Parameters can include file paths, layer and item names and text typed into text +layers, so read a repro before sharing it. + +Replay one headless (a fully wired session, like the CLI's other subcommands): + +```sh +effectcraft-cli replay crash.jsonl # per-step ok/FAIL lines, then the first failure +effectcraft-cli replay crash.jsonl --json # {ok, panicked, steps, first_failure, skipped} +effectcraft-cli replay crash.jsonl --keep-going +``` + +It exits with status 1 when a step fails, an expectation is not met or a command panics, and +stops at the first failure unless `--keep-going`. Steps that work on the user's files (`file.open` +of their project, imports) need those files; replace them with a `start` project or generated +content when you turn a report into a test. A repro only sees commands: UI state changed +without one (a hover, a half-finished drag) is not in it. + +**Regression tests are data files.** A crash fix's regression test is the repro, trimmed to the +steps that matter, with an `expect_error` or `expect` line where the fix changed the outcome, +dropped into [`tests/repros/`](../tests/repros). `crates/host/tests/repros.rs` replays every +`*.jsonl` there and fails on any panic or unmet expectation +(`cargo nextest run -p effectcraft-host --test repros`). + ## Seeing the UI To work on the UI, start the app with `cargo run -p effectcraft -- --control 9877` (add `--demo` to diff --git a/tests/repros/comp-zero-frame-rate.jsonl b/tests/repros/comp-zero-frame-rate.jsonl new file mode 100644 index 000000000..5ce9dd873 --- /dev/null +++ b/tests/repros/comp-zero-frame-rate.jsonl @@ -0,0 +1,13 @@ +// comp.new / comp.settings took a frameRate of 0 (or below 0.0005) as a 0/1 rate: the next +// frame-duration computation divided by zero and killed the process with the unsaved project +// (fixed in 780419c, "Compositions: reject zero and negative frame rates instead of panicking"). +{"start": "new"} +{"command": "comp.new", "params": {"name": "Zero", "frameRate": 0}, "expect_error": "frameRate"} +{"command": "comp.new", "params": {"name": "Tiny", "frameRate": 0.0001}, "expect_error": "frameRate"} +{"command": "comp.new", "params": {"name": "Negative", "frameRate": -24}, "expect_error": "frameRate"} +{"command": "comp.new", "params": {"name": "Main", "width": 320, "height": 180, "frameRate": 30, "duration": 1}} +{"command": "comp.settings", "params": {"frameRate": 0}, "expect_error": "frameRate"} +{"expect": {"query": "comp.info", "pointer": "/frameRate", "equals": 30}} +{"command": "layer.newSolid", "params": {"color": "#ff0000"}} +{"command": "time.set", "params": {"time": 0.5}} +{"expect_no_error": true} diff --git a/tests/repros/layer-settings-empty-selection.jsonl b/tests/repros/layer-settings-empty-selection.jsonl new file mode 100644 index 000000000..71a0216ad --- /dev/null +++ b/tests/repros/layer-settings-empty-selection.jsonl @@ -0,0 +1,8 @@ +// Layer Settings' enablement read the first selected layer with `selected_layers[0]`: with layers +// in the comp but none selected it indexed an empty list and panicked (fixed in b21e04b, M4.12). +{"start": "demo"} +{"command": "edit.deselectAll"} +{"command": "layer.settings", "expect_error": "select a layer"} +{"command": "layer.newSolid", "params": {"color": "#3366ff", "name": "Blue"}} +{"command": "layer.settings", "params": {"name": "Renamed"}} +{"expect_no_error": true} diff --git a/tests/repros/new-comp-from-selection-hostile-overlap.jsonl b/tests/repros/new-comp-from-selection-hostile-overlap.jsonl new file mode 100644 index 000000000..07e96afc6 --- /dev/null +++ b/tests/repros/new-comp-from-selection-hostile-overlap.jsonl @@ -0,0 +1,17 @@ +// File > New Comp from Selection with Sequence Layers: a huge `overlapDuration` times the layer +// count overflowed the duration arithmetic (a panic with overflow checks, a wrapped, wrong comp +// length without them), and `dimensionsFrom` past the selection indexed out of bounds +// (fixed in b21e04b, M4.12). +{"start": "new"} +{"command": "comp.new", "params": {"name": "Holder", "width": 64, "height": 64, "frameRate": 30, "duration": 2}} +{"command": "layer.newSolid", "params": {"color": "#ff0000", "name": "a"}} +{"command": "layer.newSolid", "params": {"color": "#00ff00", "name": "b"}} +{"command": "layer.newSolid", "params": {"color": "#0000ff", "name": "c"}} +{"command": "project.select", "params": {"items": ["a", "b", "c"]}} +{"command": "file.newCompFromSelection", "params": {"single": true, "sequence": true, "overlap": true, "overlapDuration": 1e30, "duration": 2}} +{"expect": {"query": "comp.info", "pointer": "/width", "equals": 64}} +{"command": "project.select", "params": {"items": ["a", "b", "c"]}} +{"command": "file.newCompFromSelection", "params": {"single": true, "sequence": true, "overlap": true, "overlapDuration": -5, "duration": 2}} +{"expect": {"query": "comp.info", "pointer": "/duration", "equals": 4}} +{"command": "project.select", "params": {"items": ["a", "b"]}} +{"command": "file.newCompFromSelection", "params": {"single": true, "dimensionsFrom": 99}, "expect_error": "dimensionsFrom"}