mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-10 18:31:30 -06:00
AG6: journal replay repros
Bug reports become replay scripts and crash regression tests become data files:
- engine::replay: a JSON-lines repro format (start line, command steps with
expect_error, expect queries with JSON pointer + equals, expect_no_error),
a never-panicking runner that reports per-step ok/error and the first failure,
and journal export (from the last project open; saves left out; size-capped).
- journal.export {path?} and help.exportRepro {path?} (not journaled).
- Crash capture: when the last-resort guard catches a panic in the outermost
command, the journal plus that command is saved to Repros/crash-*.jsonl in the
settings store (newest 20 kept) and the error names the file.
- Commands run by a journaled command (script steps) are no longer journaled twice.
- effectcraft-cli replay SCRIPT.jsonl [--json] [--keep-going]: exits 1 on a failed
step, unmet expectation or panic.
- crates/host/tests/repros.rs replays every tests/repros/*.jsonl; three samples
from past crash fixes (zero frame rate, Layer Settings with no selection,
New Comp from Selection with hostile overlap).
- Docs: docs/agents.md "Repro scripts"; AGENTS.md: crash fix regression test =
a .jsonl in tests/repros.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
(cherry picked from commit c38509d7dafe507856c48c78db65e00b94b0cebc)
(cherry picked from commit e55b56dbf76716adfcbfe4cc773df19a74edc419)
This commit is contained in:
@@ -31,6 +31,8 @@ building on top of it. Full standard: [craftrules `standards/never-crash.md`](ht
|
||||
`catch_unwind` boundary that turns an escaped panic into an error and keeps the document. It is a
|
||||
safety net, not a substitute for the rules above; keep `panic = "unwind"`.
|
||||
- **Prove it:** every crash fix comes with a small synthetic regression test that panicked before the fix.
|
||||
Crash fix regression test = drop a `.jsonl` in `tests/repros` (the crash repro, trimmed; see
|
||||
[docs/agents.md](docs/agents.md), "Repro scripts").
|
||||
|
||||
Tests, benches, fuzz targets and `xtask` may `unwrap()`; `clippy.toml` allows it in tests. Crates that are
|
||||
clean carry `#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented,
|
||||
|
||||
@@ -24,6 +24,7 @@
|
||||
//! effectcraft-cli bench --ops [--small] [--layers N] [--comps N] [--footage N] everyday-operation timings
|
||||
//! on a large generated project (open, save, auto-save, undo/redo, timeline, Project panel)
|
||||
//! effectcraft-cli script FILE.jsx [F.ecproj] | --eval CODE run an After Effects-style script
|
||||
//! effectcraft-cli replay SCRIPT.jsonl [--keep-going] replay a repro script (docs/agents.md)
|
||||
//! effectcraft-cli mcp [--autosave | --bridge PORT] MCP server on stdio
|
||||
//!
|
||||
//! Project: --project F.ecproj (or a positional *.ecproj) | --demo | --empty (default: demo; mcp: empty)
|
||||
@@ -89,7 +90,7 @@ fn stdout_broken() -> bool {
|
||||
STDOUT_FAILED.get().is_some_and(|k| *k != std::io::ErrorKind::BrokenPipe)
|
||||
}
|
||||
|
||||
const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|set|render-frame|render|script|mcp> [args] [--json]
|
||||
const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|set|render-frame|render|script|replay|mcp> [args] [--json]
|
||||
info project + engine summary
|
||||
commands [--filter TEXT] [--enabled] list engine commands
|
||||
exec <command-id> [--params JSON] run one engine command (exec --list: list them, as `commands`)
|
||||
@@ -116,6 +117,9 @@ const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|se
|
||||
script FILE.jsx [F.ecproj] | --eval CODE run JavaScript with the After Effects-style object model
|
||||
(app.project, comps, layers, properties…); prints writeLn
|
||||
output and the result; errors exit 1 with file:line:col
|
||||
replay SCRIPT.jsonl [--keep-going] replay a repro script headless (journal.export, crash repros):
|
||||
per-step ok/error and the first failure; exits 1 when a step
|
||||
fails, an expectation is not met or a command panics
|
||||
mcp [--autosave | --bridge PORT] MCP server (JSON-RPC over stdio); --autosave preserves unsaved headless work
|
||||
[--automation-read-root DIR] [--automation-write-root DIR]
|
||||
confine the agent's file access: reads under a read or write root,
|
||||
@@ -189,6 +193,7 @@ const FLAGS: &[&str] = &[
|
||||
"--small",
|
||||
"--adv3d",
|
||||
"--autosave",
|
||||
"--keep-going",
|
||||
];
|
||||
|
||||
struct Args {
|
||||
@@ -423,6 +428,56 @@ fn script_cmd(args: &Args, json_out: bool) -> Result<(), Failure> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `replay SCRIPT.jsonl`: run a repro script against a headless session (its start line picks
|
||||
/// the project) and report every step. Exits 1 when the replay fails.
|
||||
fn replay_cmd(args: &Args, json_out: bool) -> Result<(), Failure> {
|
||||
use effectcraft_engine::replay;
|
||||
let Some(file) = args.pos.first() else { return usage_err("replay needs a .jsonl script") };
|
||||
if args.opt("--bridge").is_some() || args.project.is_some() || args.flag("--demo") || args.flag("--empty") {
|
||||
return usage_err("replay runs headless from the script's start line ({\"start\": \"new\" | \"demo\" | {\"project\": path}})");
|
||||
}
|
||||
let path = std::path::Path::new(file);
|
||||
let size = std::fs::metadata(path).map_err(|e| Failure::Error(format!("cannot read {file}: {e}")))?.len();
|
||||
if size > replay::MAX_SCRIPT_BYTES as u64 {
|
||||
return Err(Failure::Error(format!("{file} is larger than {} MB", replay::MAX_SCRIPT_BYTES >> 20)));
|
||||
}
|
||||
let script = std::fs::read_to_string(path).map_err(|e| Failure::Error(format!("cannot read {file}: {e}")))?;
|
||||
let mut s = session(args)?;
|
||||
let opts = replay::Options { keep_going: args.flag("--keep-going"), base_dir: path.parent().map(std::path::Path::to_path_buf) };
|
||||
let report = replay::run(&mut s, &script, &opts);
|
||||
if json_out {
|
||||
emit(&serde_json::to_value(&report).unwrap_or_else(|e| json!({"error": e.to_string()})), true);
|
||||
} else {
|
||||
for st in &report.steps {
|
||||
let what = st.command.as_deref().unwrap_or(st.kind);
|
||||
match (&st.error, st.ok) {
|
||||
(None, _) => say!("line {:>4} ok {what}", st.line),
|
||||
(Some(e), true) => say!("line {:>4} ok {what} (tolerated error: {e})", st.line),
|
||||
(Some(e), false) => say!("line {:>4} FAIL {what}: {e}", st.line),
|
||||
}
|
||||
}
|
||||
}
|
||||
let Some(first) = &report.first_failure else {
|
||||
if !json_out {
|
||||
say!("replay ok: {} steps", report.steps.len());
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
let msg = format!(
|
||||
"failed at line {} ({}){}: {}{}",
|
||||
first.line,
|
||||
first.command.as_deref().unwrap_or(first.kind),
|
||||
if report.panicked { ", a command panicked" } else { "" },
|
||||
first.error.as_deref().unwrap_or(""),
|
||||
if report.skipped > 0 { format!(" ({} later steps not run)", report.skipped) } else { String::new() }
|
||||
);
|
||||
if json_out {
|
||||
note!("effectcraft-cli replay: {msg}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
Err(Failure::Error(msg))
|
||||
}
|
||||
|
||||
/// Print a result: compact JSON with `--json`, else pretty JSON.
|
||||
fn emit(v: &Value, json_out: bool) {
|
||||
if json_out {
|
||||
@@ -485,6 +540,7 @@ fn run(cmd: &str, args: &Args, json_out: bool) -> Result<(), Failure> {
|
||||
"commands" => list_commands_cmd(args, json_out)?,
|
||||
"exec" if args.flag("--list") => list_commands_cmd(args, json_out)?,
|
||||
"script" => script_cmd(args, json_out)?,
|
||||
"replay" => replay_cmd(args, json_out)?,
|
||||
"exec" => {
|
||||
let Some(id) = args.pos.first().cloned() else { return usage_err("exec needs a command id") };
|
||||
let params = match (args.opt("--params"), args.pos.get(1).map(String::as_str)) {
|
||||
|
||||
@@ -491,3 +491,31 @@ fn render_bitrate_applies_to_vp9_webm() {
|
||||
assert!(high > low, "100 kbps: {low} bytes, 8000 kbps: {high} bytes");
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// `replay`: a repro script runs headless with per-step results; a failing step exits 1 and is
|
||||
/// named as the first failure; malformed lines are failures, not crashes.
|
||||
#[test]
|
||||
fn replay_reports_steps_and_the_first_failure() {
|
||||
let repro = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../tests/repros/comp-zero-frame-rate.jsonl");
|
||||
let v = ok_json(&["replay", repro.to_str().unwrap()]);
|
||||
assert_eq!(v["ok"], true, "{v}");
|
||||
assert!(v["steps"].as_array().unwrap().len() > 5);
|
||||
let bad = tmp("bad-repro.jsonl");
|
||||
std::fs::write(
|
||||
&bad,
|
||||
"{\"start\": \"new\"}\n{\"command\": \"comp.new\"}\n{\"expect\": {\"query\": \"comp.info\", \"pointer\": \"/width\", \"equals\": 1}}\nnot json\n",
|
||||
)
|
||||
.unwrap();
|
||||
let (code, v) = run_json(&["replay", bad.to_str().unwrap()]);
|
||||
assert_eq!(code, 1);
|
||||
assert_eq!(v["ok"], false);
|
||||
assert_eq!(v["first_failure"]["line"], 3, "{v}");
|
||||
assert_eq!(v["skipped"], 1);
|
||||
let out = bin().args(["replay", bad.to_str().unwrap(), "--keep-going"]).output().unwrap();
|
||||
assert_eq!(out.status.code(), Some(1));
|
||||
let text = String::from_utf8_lossy(&out.stdout);
|
||||
assert!(text.contains("line 2 ok comp.new") && text.contains("line 4 FAIL invalid: not JSON"), "{text}");
|
||||
assert!(String::from_utf8_lossy(&out.stderr).contains("failed at line 3 (comp.info)"));
|
||||
// The start line picks the project.
|
||||
assert_eq!(bin().args(["replay", bad.to_str().unwrap(), "--demo"]).output().unwrap().status.code(), Some(2));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
//! The command journal as a replay script (see [`crate::replay`]): `journal.export` for agents
|
||||
//! and `help.exportRepro` for bug reports. Neither is journaled.
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::{CommandSpec, always, str_p};
|
||||
use crate::{Result, Session, replay};
|
||||
|
||||
/// `journal.export {path?}`: the script itself, or written to `path`.
|
||||
fn export(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
if let Some(path) = str_p(p, "path") {
|
||||
let (path, steps) = replay::save_repro(s, Some(path))?;
|
||||
return Ok(json!({"path": path, "steps": steps}));
|
||||
}
|
||||
let script = replay::export(s, None);
|
||||
Ok(json!({"steps": script.lines().count().saturating_sub(1), "script": script}))
|
||||
}
|
||||
|
||||
/// `help.exportRepro {path?}`: save the steps so far for a bug report (default: the settings
|
||||
/// folder's `Repros`).
|
||||
fn export_repro(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let (path, steps) = replay::save_repro(s, str_p(p, "path"))?;
|
||||
Ok(json!({"path": path, "steps": steps}))
|
||||
}
|
||||
|
||||
const fn spec(id: &'static str, label: &'static str, params: &'static str, run: super::Run) -> CommandSpec {
|
||||
CommandSpec { id, label, menu: &[], shortcut: None, params, enabled: always, run, journal: false }
|
||||
}
|
||||
|
||||
pub fn specs() -> Vec<CommandSpec> {
|
||||
#[allow(unused_mut)]
|
||||
let mut v = vec![
|
||||
spec(
|
||||
"journal.export",
|
||||
"Export Command Journal",
|
||||
"{path? (write the script there)} → {script, steps} or {path, steps}: the commands run since the project was last opened, as a replay script (docs/agents.md, Repro scripts)",
|
||||
export,
|
||||
),
|
||||
spec(
|
||||
"help.exportRepro",
|
||||
"Save Bug Repro...",
|
||||
"{path? (default: Repros in the settings folder)} → {path, steps}: the steps since the project was opened, for a bug report (commands and parameters only)",
|
||||
export_repro,
|
||||
),
|
||||
];
|
||||
// A command with a bug, for the crash-repro tests.
|
||||
#[cfg(test)]
|
||||
v.push(spec("test.panic", "Panic (test)", "{why?}", |_, p| {
|
||||
let why = str_p(p, "why").unwrap_or("test").to_string();
|
||||
std::panic::panic_any(why)
|
||||
}));
|
||||
v
|
||||
}
|
||||
@@ -25,6 +25,7 @@ pub mod footage_panel;
|
||||
pub(crate) mod frame_export;
|
||||
mod frontend;
|
||||
mod help;
|
||||
mod journal;
|
||||
mod key_labels;
|
||||
mod key_transform;
|
||||
mod keys_more;
|
||||
@@ -157,6 +158,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
|
||||
v.extend(time::specs());
|
||||
v.extend(render_queue::specs());
|
||||
v.extend(help::specs());
|
||||
v.extend(journal::specs());
|
||||
v.extend(query::specs());
|
||||
v.extend(batch::specs());
|
||||
v.extend(layer_menu::specs());
|
||||
|
||||
@@ -4,14 +4,28 @@
|
||||
//! panics leaves the project as it was. This is a safety net, not a substitute for returning
|
||||
//! errors (AGENTS.md, "Never crash").
|
||||
|
||||
use std::cell::Cell;
|
||||
|
||||
use crate::{EngineError, Result};
|
||||
|
||||
thread_local! {
|
||||
/// Panics [`guarded`] caught on this thread (see [`caught`]).
|
||||
static CAUGHT: Cell<u64> = const { Cell::new(0) };
|
||||
}
|
||||
|
||||
/// How many panics [`guarded`] has caught on this thread so far: compare two readings to learn
|
||||
/// whether a call panicked (crash repros, `effectcraft-cli replay`).
|
||||
pub fn caught() -> u64 {
|
||||
CAUGHT.with(Cell::get)
|
||||
}
|
||||
|
||||
/// Run `f`; a panic inside it becomes an error naming `what`. The panic itself is logged by
|
||||
/// the panic hook ([`crate::logging::install_panic_hook`]).
|
||||
pub fn guarded<T>(what: &str, f: impl FnOnce() -> Result<T>) -> Result<T> {
|
||||
match std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)) {
|
||||
Ok(r) => r,
|
||||
Err(payload) => {
|
||||
CAUGHT.with(|c| c.set(c.get().saturating_add(1)));
|
||||
let msg = payload
|
||||
.downcast_ref::<&str>()
|
||||
.map(|s| s.to_string())
|
||||
|
||||
@@ -35,6 +35,7 @@ pub mod preview;
|
||||
pub mod psd_import;
|
||||
pub mod remote;
|
||||
pub mod render_queue;
|
||||
pub mod replay;
|
||||
pub mod roto;
|
||||
pub mod scriptui;
|
||||
pub mod sequence;
|
||||
@@ -598,10 +599,15 @@ impl Session {
|
||||
return Err(EngineError::Disabled(id.to_string(), why));
|
||||
}
|
||||
}
|
||||
// Last-resort guard: a panicking command reports an error instead of crashing the app.
|
||||
let r = guard::guarded(&format!("command `{id}`"), || (spec.run)(self, ¶ms))?;
|
||||
// Last-resort guard: a panicking command reports an error instead of crashing the app
|
||||
// (and saves a repro of the steps that led there, see `replay::after_command`).
|
||||
let nesting = replay::Nesting::enter(spec.journal);
|
||||
let caught = guard::caught();
|
||||
let r = guard::guarded(&format!("command `{id}`"), || (spec.run)(self, ¶ms));
|
||||
let r = replay::after_command(self, id, ¶ms, &nesting, caught, r)?;
|
||||
self.learn_observe(id, ¶ms);
|
||||
if spec.journal {
|
||||
// Commands run by a journaled command (a script's steps) replay with it, not again.
|
||||
if spec.journal && !nesting.inside_journaled() {
|
||||
self.journal.push((id.to_string(), params));
|
||||
if self.journal.len() > 10_000 {
|
||||
self.journal.drain(..1000);
|
||||
|
||||
@@ -0,0 +1,693 @@
|
||||
//! Replay scripts: a bug or crash as a list of commands an agent (or a test) can run again.
|
||||
//!
|
||||
//! The session's command journal ([`Session::journal`]) becomes a JSON-lines script
|
||||
//! (`journal.export`, `help.exportRepro`, and automatically when the last-resort guard catches a
|
||||
//! panic, see [`after_command`]); `effectcraft-cli replay script.jsonl` runs one headless and
|
||||
//! `crates/host/tests/repros.rs` runs every `tests/repros/*.jsonl` as a regression test.
|
||||
//!
|
||||
//! Format (one JSON object per line; blank lines and lines starting with `//` or `#` are
|
||||
//! comments):
|
||||
//!
|
||||
//! ```text
|
||||
//! {"start": "new" | "demo" | {"project": "path.ecproj"}, "allow_errors"?: bool} optional, first
|
||||
//! {"command": "comp.new", "params": {...}, "expect_error"?: true | "substring"}
|
||||
//! {"expect": {"query": "prop.get", "params": {...}, "pointer"?: "/value", "equals"?: ...}}
|
||||
//! {"expect_no_error": true}
|
||||
//! ```
|
||||
//!
|
||||
//! A command must succeed unless it says `expect_error` (then it must fail, with that text in
|
||||
//! the message when it is a string), or the start line says `allow_errors` (errors are reported
|
||||
//! but tolerated; crash repros say so, since after the fix the crashing step may fail cleanly).
|
||||
//! A panic always fails. `expect` runs a command (normally a query) and compares the value at
|
||||
//! the JSON `pointer` of its result with `equals` (numbers within 1e-6); `expect_no_error`
|
||||
//! checks that the previous command succeeded. Other keys (`version`, `app`, `note`, `crash`…)
|
||||
//! are information for people. Relative `project` paths are relative to the script.
|
||||
//!
|
||||
//! Privacy: a repro holds command ids and their parameters only, never project or media file
|
||||
//! contents. Parameters can include file paths, layer names and text typed into text layers.
|
||||
|
||||
use std::cell::Cell;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::{EngineError, Result, Session, guard};
|
||||
|
||||
/// The format version written into exported headers.
|
||||
pub const VERSION: u64 = 1;
|
||||
/// Largest script [`run`] accepts (bytes).
|
||||
pub const MAX_SCRIPT_BYTES: usize = 64 << 20;
|
||||
/// Most lines [`run`] reads.
|
||||
pub const MAX_LINES: usize = 100_000;
|
||||
/// Exported scripts stay below this size: the oldest steps are dropped (and counted).
|
||||
const MAX_EXPORT_BYTES: usize = 32 << 20;
|
||||
/// Folder of saved repros in the settings store (next to `Logs`).
|
||||
pub const REPRO_DIR: &str = "Repros";
|
||||
/// Automatic crash repros kept (the oldest are removed).
|
||||
const KEEP_CRASH_REPROS: usize = 20;
|
||||
/// Commands that replace the whole project: an export starts at the last one.
|
||||
const PROJECT_STARTS: &[&str] = &["file.newProject", "file.open", "file.openDemoProject"];
|
||||
/// Error and value texts in reports are cut to this many characters.
|
||||
const MAX_TEXT: usize = 600;
|
||||
|
||||
// ---------- nesting and crash capture (Session::execute) ----------
|
||||
|
||||
thread_local! {
|
||||
/// Commands running on this thread: (all, journaled ones).
|
||||
static DEPTH: Cell<(u32, u32)> = const { Cell::new((0, 0)) };
|
||||
/// A replay is running on this thread (its panics are reported, not saved as repros).
|
||||
static REPLAYING: Cell<bool> = const { Cell::new(false) };
|
||||
}
|
||||
|
||||
/// One running command (see [`Session::execute`]): commands it runs itself (a script's steps)
|
||||
/// aren't journaled again, and only the outermost command saves a crash repro.
|
||||
pub struct Nesting {
|
||||
outer: (u32, u32),
|
||||
}
|
||||
|
||||
impl Nesting {
|
||||
pub fn enter(journaled: bool) -> Nesting {
|
||||
let outer = DEPTH.with(Cell::get);
|
||||
DEPTH.with(|d| d.set((outer.0.saturating_add(1), outer.1.saturating_add(u32::from(journaled)))));
|
||||
Nesting { outer }
|
||||
}
|
||||
/// Run by a journaled command (which replays it).
|
||||
pub fn inside_journaled(&self) -> bool {
|
||||
self.outer.1 > 0
|
||||
}
|
||||
pub fn outermost(&self) -> bool {
|
||||
self.outer.0 == 0
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Nesting {
|
||||
fn drop(&mut self) {
|
||||
DEPTH.with(|d| d.set(self.outer));
|
||||
}
|
||||
}
|
||||
|
||||
/// After the outermost command: when the guard caught a panic while it ran (`caught` is
|
||||
/// [`guard::caught`] from before), save the journal plus this command as a crash repro in the
|
||||
/// settings store and name it in the error.
|
||||
pub fn after_command(s: &Session, id: &str, params: &Value, nesting: &Nesting, caught: u64, r: Result<Value>) -> Result<Value> {
|
||||
if !nesting.outermost() || guard::caught() == caught || REPLAYING.with(Cell::get) {
|
||||
return r;
|
||||
}
|
||||
let error = match &r {
|
||||
Err(e) => e.to_string(),
|
||||
Ok(_) => "a step it ran failed with an internal error".to_string(),
|
||||
};
|
||||
let saved = save_crash_repro(s, id, params, &error);
|
||||
match (r, saved) {
|
||||
(Err(EngineError::Other(m)), Some(path)) => Err(EngineError::Other(format!("{m} (the steps that led here were saved to {path})"))),
|
||||
(r, _) => r,
|
||||
}
|
||||
}
|
||||
|
||||
fn unix_secs() -> u64 {
|
||||
web_time::SystemTime::now().duration_since(web_time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// `Repros/<prefix>-<seconds>-<command>.jsonl`.
|
||||
fn repro_name(prefix: &str, id: &str) -> String {
|
||||
let id: String = id.chars().map(|c| if c.is_ascii_alphanumeric() || c == '.' { c } else { '_' }).take(60).collect();
|
||||
format!("{REPRO_DIR}/{prefix}-{:012}-{id}.jsonl", unix_secs())
|
||||
}
|
||||
|
||||
/// Where a store entry is on disk (or its name in stores without a folder).
|
||||
fn store_path(store: &dyn crate::config::ConfigStore, name: &str) -> String {
|
||||
store.dir().map(|d| d.join(name).display().to_string()).unwrap_or_else(|| name.to_string())
|
||||
}
|
||||
|
||||
fn save_crash_repro(s: &Session, id: &str, params: &Value, error: &str) -> Option<String> {
|
||||
let store = s.config.clone()?;
|
||||
let mut text = export(s, Some(json!({"command": id, "error": cut(error)})));
|
||||
text.push_str(&command_line(id, params));
|
||||
text.push('\n');
|
||||
let name = repro_name("crash", id);
|
||||
if let Err(e) = store.write(&name, &text) {
|
||||
log::error!("cannot save the crash repro {name}: {e}");
|
||||
return None;
|
||||
}
|
||||
// Keep the newest few (names sort by time).
|
||||
let mut old: Vec<String> = store.list(REPRO_DIR).into_iter().filter(|n| n.starts_with("crash-")).collect();
|
||||
old.sort();
|
||||
let extra = old.len().saturating_sub(KEEP_CRASH_REPROS);
|
||||
for n in old.into_iter().take(extra) {
|
||||
let _ = store.remove(&format!("{REPRO_DIR}/{n}"));
|
||||
}
|
||||
let path = store_path(store.as_ref(), &name);
|
||||
log::error!("crash repro saved to {path}");
|
||||
Some(path)
|
||||
}
|
||||
|
||||
/// Save the journal as a repro: at `path` (through the session's file services), else in the
|
||||
/// settings store's `Repros` folder. Returns where it went and how many steps it holds.
|
||||
pub fn save_repro(s: &Session, path: Option<&str>) -> Result<(String, usize)> {
|
||||
let text = export(s, None);
|
||||
let steps = text.lines().count().saturating_sub(1);
|
||||
if let Some(path) = path {
|
||||
s.services.write_file(path, text.as_bytes()).map_err(|e| EngineError::Other(format!("cannot write {path}: {e}")))?;
|
||||
return Ok((path.to_string(), steps));
|
||||
}
|
||||
let store = s.config.clone().ok_or_else(|| EngineError::Other("there is no settings folder to save the repro in; pass `path`".into()))?;
|
||||
let name = repro_name("repro", "session");
|
||||
store.write(&name, &text).map_err(|e| EngineError::Other(format!("cannot save {name}: {e}")))?;
|
||||
Ok((store_path(store.as_ref(), &name), steps))
|
||||
}
|
||||
|
||||
// ---------- export ----------
|
||||
|
||||
fn command_line(id: &str, params: &Value) -> String {
|
||||
let mut o = Map::new();
|
||||
o.insert("command".into(), json!(id));
|
||||
if !(params.is_null() || params.as_object().is_some_and(Map::is_empty)) {
|
||||
o.insert("params".into(), params.clone());
|
||||
}
|
||||
Value::Object(o).to_string()
|
||||
}
|
||||
|
||||
/// Saving writes the user's files and doesn't change the project: left out of repros.
|
||||
fn skipped_in_export(id: &str) -> bool {
|
||||
id.starts_with("file.save") || id.starts_with("file.increment")
|
||||
}
|
||||
|
||||
/// The journal as a replay script: a start line, then the steps since the project was last
|
||||
/// replaced (`file.newProject` / `file.open` / `file.openDemoProject`). `crash` (the failing
|
||||
/// command and its error) marks a crash repro, which tolerates errors on replay.
|
||||
pub fn export(s: &Session, crash: Option<Value>) -> String {
|
||||
let from = s.journal.iter().rposition(|(id, _)| PROJECT_STARTS.contains(&id.as_str())).unwrap_or(0);
|
||||
let mut lines: Vec<String> =
|
||||
s.journal.get(from..).unwrap_or_default().iter().filter(|(id, _)| !skipped_in_export(id)).map(|(id, p)| command_line(id, p)).collect();
|
||||
let mut size: usize = lines.iter().map(|l| l.len().saturating_add(1)).sum();
|
||||
let mut dropped = 0usize;
|
||||
while size > MAX_EXPORT_BYTES && !lines.is_empty() {
|
||||
size = size.saturating_sub(lines.remove(0).len().saturating_add(1));
|
||||
dropped += 1;
|
||||
}
|
||||
let mut head = Map::new();
|
||||
head.insert("start".into(), json!("new"));
|
||||
head.insert("version".into(), json!(VERSION));
|
||||
head.insert("app".into(), json!(format!("EffectCraft {}", env!("CARGO_PKG_VERSION"))));
|
||||
head.insert("os".into(), json!(std::env::consts::OS));
|
||||
head.insert("exported".into(), json!(unix_secs()));
|
||||
if let Some(c) = crash {
|
||||
head.insert("allow_errors".into(), json!(true));
|
||||
head.insert("crash".into(), c);
|
||||
}
|
||||
if dropped > 0 {
|
||||
head.insert("dropped".into(), json!(dropped));
|
||||
}
|
||||
head.insert("note".into(), json!("commands and parameters only (no file contents); parameters can include file paths and text"));
|
||||
let mut out = Value::Object(head).to_string();
|
||||
out.push('\n');
|
||||
for l in lines {
|
||||
out.push_str(&l);
|
||||
out.push('\n');
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// ---------- parsing ----------
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum Start {
|
||||
New,
|
||||
Demo,
|
||||
Project(String),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum Step {
|
||||
Start(Start),
|
||||
Command {
|
||||
id: String,
|
||||
params: Value,
|
||||
expect_error: Option<Option<String>>,
|
||||
},
|
||||
Expect {
|
||||
query: String,
|
||||
params: Value,
|
||||
pointer: Option<String>,
|
||||
equals: Option<Value>,
|
||||
},
|
||||
ExpectNoError,
|
||||
/// A line that isn't a valid step (reported as a failure).
|
||||
Invalid(String),
|
||||
}
|
||||
|
||||
fn params_of(v: Option<&Value>) -> std::result::Result<Value, String> {
|
||||
match v {
|
||||
None | Some(Value::Null) => Ok(json!({})),
|
||||
Some(p @ Value::Object(_)) => Ok(p.clone()),
|
||||
Some(_) => Err("`params` must be an object".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_start(v: &Value) -> std::result::Result<Start, String> {
|
||||
match v {
|
||||
Value::String(s) if s == "new" => Ok(Start::New),
|
||||
Value::String(s) if s == "demo" => Ok(Start::Demo),
|
||||
Value::Object(o) => match o.get("project").and_then(Value::as_str) {
|
||||
Some(p) if !p.trim().is_empty() => Ok(Start::Project(p.to_string())),
|
||||
_ => Err("`start` object needs a `project` path".into()),
|
||||
},
|
||||
_ => Err("`start` must be \"new\", \"demo\" or {\"project\": path}".into()),
|
||||
}
|
||||
}
|
||||
|
||||
/// One line → a step (`None`: a blank or comment line).
|
||||
fn parse_line(text: &str, first: bool, allow_errors: &mut bool) -> Option<Step> {
|
||||
let t = text.trim();
|
||||
if t.is_empty() || t.starts_with("//") || t.starts_with('#') {
|
||||
return None;
|
||||
}
|
||||
let v: Value = match serde_json::from_str(t) {
|
||||
Ok(v) => v,
|
||||
Err(e) => return Some(Step::Invalid(format!("not JSON: {e}"))),
|
||||
};
|
||||
let Some(o) = v.as_object() else { return Some(Step::Invalid("a line must be a JSON object".into())) };
|
||||
let step = if let Some(st) = o.get("start") {
|
||||
if !first {
|
||||
return Some(Step::Invalid("the `start` line must come first".into()));
|
||||
}
|
||||
*allow_errors = o.get("allow_errors").and_then(Value::as_bool).unwrap_or(false);
|
||||
parse_start(st).map(Step::Start)
|
||||
} else if let Some(c) = o.get("command") {
|
||||
let Some(id) = c.as_str().filter(|s| !s.is_empty()) else { return Some(Step::Invalid("`command` must be a command id".into())) };
|
||||
let expect_error = match o.get("expect_error") {
|
||||
None | Some(Value::Null | Value::Bool(false)) => Ok(None),
|
||||
Some(Value::Bool(true)) => Ok(Some(None)),
|
||||
Some(Value::String(s)) => Ok(Some(Some(s.clone()))),
|
||||
Some(_) => Err("`expect_error` must be true or a text the error contains".to_string()),
|
||||
};
|
||||
match (params_of(o.get("params")), expect_error) {
|
||||
(Ok(params), Ok(expect_error)) => Ok(Step::Command { id: id.to_string(), params, expect_error }),
|
||||
(Err(m), _) | (_, Err(m)) => Err(m),
|
||||
}
|
||||
} else if let Some(e) = o.get("expect") {
|
||||
let query = e.get("query").and_then(Value::as_str).filter(|s| !s.is_empty());
|
||||
let pointer = e.get("pointer").map(|p| p.as_str().map(str::to_string).ok_or("`pointer` must be a JSON pointer such as /value"));
|
||||
match (query, pointer.transpose()) {
|
||||
(None, _) => Err("`expect` needs a `query` command id".to_string()),
|
||||
(_, Err(m)) => Err(m.to_string()),
|
||||
(Some(q), Ok(pointer)) => {
|
||||
params_of(e.get("params")).map(|params| Step::Expect { query: q.to_string(), params, pointer, equals: e.get("equals").cloned() })
|
||||
}
|
||||
}
|
||||
} else if o.get("expect_no_error").and_then(Value::as_bool) == Some(true) {
|
||||
Ok(Step::ExpectNoError)
|
||||
} else if o.len() == 1 && o.contains_key("comment") {
|
||||
return None;
|
||||
} else {
|
||||
Err("expected a `start`, `command`, `expect` or `expect_no_error` line".to_string())
|
||||
};
|
||||
Some(step.unwrap_or_else(Step::Invalid))
|
||||
}
|
||||
|
||||
// ---------- running ----------
|
||||
|
||||
/// How to run a script.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct Options {
|
||||
/// Keep running after a failed step (default: stop at the first failure).
|
||||
pub keep_going: bool,
|
||||
/// Folder relative `start.project` paths are resolved against (the script's).
|
||||
pub base_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
/// What happened to one line.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct StepReport {
|
||||
/// 1-based line number (0: the implicit start).
|
||||
pub line: usize,
|
||||
/// `start`, `command`, `expect`, `expect_no_error` or `invalid`.
|
||||
pub kind: &'static str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub command: Option<String>,
|
||||
pub ok: bool,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
#[serde(skip_serializing_if = "std::ops::Not::not")]
|
||||
pub panicked: bool,
|
||||
}
|
||||
|
||||
/// The outcome of [`run`].
|
||||
#[derive(Clone, Debug, Default, Serialize)]
|
||||
pub struct Report {
|
||||
pub ok: bool,
|
||||
/// A step panicked (caught by the last-resort guard): always a failure.
|
||||
pub panicked: bool,
|
||||
pub steps: Vec<StepReport>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub first_failure: Option<StepReport>,
|
||||
/// Steps not run after the first failure.
|
||||
pub skipped: usize,
|
||||
}
|
||||
|
||||
/// Cut a message to [`MAX_TEXT`] characters.
|
||||
fn cut(s: &str) -> String {
|
||||
match s.char_indices().nth(MAX_TEXT) {
|
||||
Some((i, _)) => format!("{}…", s.get(..i).unwrap_or(s)),
|
||||
None => s.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Run one command the way the app does, noting whether it panicked.
|
||||
fn exec(s: &mut Session, id: &str, params: Value) -> (Result<Value>, bool) {
|
||||
let before = guard::caught();
|
||||
let r = guard::guarded(&format!("replay of `{id}`"), || s.execute(id, params));
|
||||
s.events.clear();
|
||||
(r, guard::caught() != before)
|
||||
}
|
||||
|
||||
/// Numbers within 1e-6 (relative above 1), everything else exactly.
|
||||
fn approx_eq(a: &Value, b: &Value) -> bool {
|
||||
match (a, b) {
|
||||
(Value::Number(x), Value::Number(y)) => match (x.as_f64(), y.as_f64()) {
|
||||
(Some(x), Some(y)) => (x - y).abs() <= 1e-6 * x.abs().max(y.abs()).max(1.0),
|
||||
_ => x == y,
|
||||
},
|
||||
(Value::Array(x), Value::Array(y)) => x.len() == y.len() && x.iter().zip(y).all(|(a, b)| approx_eq(a, b)),
|
||||
(Value::Object(x), Value::Object(y)) => x.len() == y.len() && x.iter().all(|(k, a)| y.get(k).is_some_and(|b| approx_eq(a, b))),
|
||||
_ => a == b,
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks this thread as replaying while alive.
|
||||
struct Replaying(bool);
|
||||
|
||||
impl Replaying {
|
||||
fn enter() -> Replaying {
|
||||
Replaying(REPLAYING.with(|r| r.replace(true)))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Replaying {
|
||||
fn drop(&mut self) {
|
||||
REPLAYING.with(|r| r.set(self.0));
|
||||
}
|
||||
}
|
||||
|
||||
fn start_step(s: &mut Session, start: &Start, base: Option<&Path>) -> (String, Result<Value>, bool) {
|
||||
match start {
|
||||
Start::New => {
|
||||
let (r, p) = exec(s, "file.newProject", json!({}));
|
||||
("file.newProject".into(), r, p)
|
||||
}
|
||||
Start::Demo => {
|
||||
let (r, p) = exec(s, "file.openDemoProject", json!({}));
|
||||
("file.openDemoProject".into(), r, p)
|
||||
}
|
||||
Start::Project(path) => {
|
||||
let full = match base {
|
||||
Some(b) if Path::new(path).is_relative() => b.join(path).display().to_string(),
|
||||
_ => path.clone(),
|
||||
};
|
||||
let (r, p) = exec(s, "file.open", json!({"path": full}));
|
||||
("file.open".into(), r, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run a replay script against `s`. Never panics: malformed lines are failed steps.
|
||||
pub fn run(s: &mut Session, script: &str, opts: &Options) -> Report {
|
||||
let _replaying = Replaying::enter();
|
||||
let mut report = Report { ok: true, ..Default::default() };
|
||||
let fail = |report: &mut Report, sr: StepReport| {
|
||||
report.ok = false;
|
||||
report.panicked |= sr.panicked;
|
||||
if report.first_failure.is_none() {
|
||||
report.first_failure = Some(sr.clone());
|
||||
}
|
||||
report.steps.push(sr);
|
||||
};
|
||||
if script.len() > MAX_SCRIPT_BYTES {
|
||||
let msg = format!("the script is larger than {} MB", MAX_SCRIPT_BYTES >> 20);
|
||||
fail(&mut report, StepReport { line: 0, kind: "invalid", command: None, ok: false, error: Some(msg), panicked: false });
|
||||
return report;
|
||||
}
|
||||
let mut allow_errors = false;
|
||||
let mut steps: Vec<(usize, Step)> = vec![];
|
||||
for (i, line) in script.lines().enumerate().take(MAX_LINES) {
|
||||
if let Some(step) = parse_line(line, steps.is_empty(), &mut allow_errors) {
|
||||
steps.push((i + 1, step));
|
||||
}
|
||||
}
|
||||
if script.lines().nth(MAX_LINES).is_some() {
|
||||
steps.push((MAX_LINES + 1, Step::Invalid(format!("the script has more than {MAX_LINES} lines; the rest was not read"))));
|
||||
}
|
||||
if !matches!(steps.first(), Some((_, Step::Start(_)) | (_, Step::Invalid(_)))) {
|
||||
steps.insert(0, (0, Step::Start(Start::New)));
|
||||
}
|
||||
let mut last_command_failed: Option<bool> = None;
|
||||
let mut stopped = false;
|
||||
for (line, step) in steps {
|
||||
if stopped {
|
||||
report.skipped += 1;
|
||||
continue;
|
||||
}
|
||||
let mut sr = StepReport { line, kind: "command", command: None, ok: true, error: None, panicked: false };
|
||||
match step {
|
||||
Step::Invalid(m) => {
|
||||
sr.kind = "invalid";
|
||||
sr.ok = false;
|
||||
sr.error = Some(m);
|
||||
}
|
||||
Step::Start(start) => {
|
||||
sr.kind = "start";
|
||||
let (id, r, panicked) = start_step(s, &start, opts.base_dir.as_deref());
|
||||
sr.command = Some(id);
|
||||
sr.panicked = panicked;
|
||||
if let Err(e) = r {
|
||||
sr.ok = false;
|
||||
sr.error = Some(cut(&e.to_string()));
|
||||
}
|
||||
}
|
||||
Step::Command { id, params, expect_error } => {
|
||||
let (r, panicked) = exec(s, &id, params);
|
||||
sr.command = Some(id);
|
||||
sr.panicked = panicked;
|
||||
last_command_failed = Some(r.is_err());
|
||||
let err = r.err().map(|e| e.to_string());
|
||||
let (ok, error) = match (err, expect_error) {
|
||||
(e, _) if panicked => (false, Some(format!("panicked: {}", e.unwrap_or_else(|| "a step failed with an internal error".into())))),
|
||||
(None, None) => (true, None),
|
||||
(None, Some(_)) => (false, Some("expected an error, but the command succeeded".into())),
|
||||
(Some(e), Some(Some(want))) if !e.contains(want.as_str()) => (false, Some(format!("expected an error containing {want:?}, got: {e}"))),
|
||||
(Some(_), Some(_)) => (true, None),
|
||||
(Some(e), None) => (allow_errors, Some(e)),
|
||||
};
|
||||
sr.ok = ok;
|
||||
sr.error = error.as_deref().map(cut);
|
||||
}
|
||||
Step::Expect { query, params, pointer, equals } => {
|
||||
sr.kind = "expect";
|
||||
let (r, panicked) = exec(s, &query, params);
|
||||
sr.command = Some(query);
|
||||
sr.panicked = panicked;
|
||||
match r {
|
||||
Err(e) => {
|
||||
sr.ok = false;
|
||||
sr.error = Some(cut(&format!("{}{e}", if panicked { "panicked: " } else { "" })));
|
||||
}
|
||||
Ok(v) => {
|
||||
let got = match &pointer {
|
||||
Some(p) => v.pointer(p),
|
||||
None => Some(&v),
|
||||
};
|
||||
match (got, &equals) {
|
||||
(None, _) => {
|
||||
sr.ok = false;
|
||||
sr.error = Some(cut(&format!("no value at {} in {v}", pointer.as_deref().unwrap_or(""))));
|
||||
}
|
||||
(Some(g), Some(want)) if !approx_eq(g, want) => {
|
||||
sr.ok = false;
|
||||
sr.error = Some(cut(&format!("expected {want}, got {g}")));
|
||||
}
|
||||
_ => sr.ok = !panicked,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Step::ExpectNoError => {
|
||||
sr.kind = "expect_no_error";
|
||||
match last_command_failed {
|
||||
Some(false) => {}
|
||||
Some(true) => {
|
||||
sr.ok = false;
|
||||
sr.error = Some("the previous command failed".into());
|
||||
}
|
||||
None => {
|
||||
sr.ok = false;
|
||||
sr.error = Some("no command before this line".into());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if sr.ok {
|
||||
report.steps.push(sr);
|
||||
} else {
|
||||
fail(&mut report, sr);
|
||||
stopped = !opts.keep_going;
|
||||
}
|
||||
}
|
||||
report
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::json;
|
||||
|
||||
use super::{Options, run};
|
||||
use crate::Session;
|
||||
use crate::config::{ConfigStore, MemoryConfig};
|
||||
|
||||
fn replay(script: &str) -> super::Report {
|
||||
run(&mut Session::new(), script, &Options::default())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn steps_expectations_and_failures_are_reported() {
|
||||
let r = replay(
|
||||
r#"{"start": "new"}
|
||||
// a comment
|
||||
{"command": "comp.new", "params": {"name": "A", "width": 320, "height": 180, "frameRate": 30, "duration": 2}}
|
||||
{"expect_no_error": true}
|
||||
{"command": "comp.new", "params": {"frameRate": 0}, "expect_error": "frame"}
|
||||
{"expect": {"query": "comp.info", "pointer": "/width", "equals": 320}}
|
||||
"#,
|
||||
);
|
||||
assert!(r.ok, "{r:?}");
|
||||
assert_eq!(r.steps.len(), 5);
|
||||
let r = replay(
|
||||
"{\"command\": \"comp.new\"}\n{\"expect\": {\"query\": \"comp.info\", \"pointer\": \"/width\", \"equals\": 1}}\n{\"command\": \"comp.new\"}",
|
||||
);
|
||||
assert!(!r.ok && !r.panicked && r.skipped == 1, "{r:?}");
|
||||
assert_eq!(r.first_failure.as_ref().map(|f| f.line), Some(2));
|
||||
// An unexpected success, an unknown command.
|
||||
assert!(!replay(r#"{"command": "comp.new", "expect_error": true}"#).ok);
|
||||
assert!(!replay(r#"{"command": "no.such.command"}"#).ok);
|
||||
// Tolerated errors.
|
||||
let r = replay("{\"start\": \"new\", \"allow_errors\": true}\n{\"command\": \"no.such.command\"}\n{\"expect_no_error\": true}");
|
||||
assert!(!r.ok && r.first_failure.as_ref().map(|f| f.kind) == Some("expect_no_error"), "{r:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_lines_are_failures_not_crashes() {
|
||||
for bad in [
|
||||
"not json",
|
||||
"[1, 2]",
|
||||
"{\"command\": 5}",
|
||||
"{\"command\": \"comp.new\", \"params\": [1]}",
|
||||
"{\"command\": \"comp.new\", \"expect_error\": 3}",
|
||||
"{\"expect\": {}}",
|
||||
"{\"expect\": {\"query\": \"comp.info\", \"pointer\": 4}}",
|
||||
"{\"expect\": {\"query\": \"comp.info\", \"pointer\": \"/nope\"}}",
|
||||
"{\"start\": 7}",
|
||||
"{\"start\": {\"project\": \"\"}}",
|
||||
"{\"command\": \"comp.new\"}\n{\"start\": \"new\"}",
|
||||
"{\"what\": 1}",
|
||||
"{\"expect_no_error\": true}",
|
||||
"{\"start\": {\"project\": \"/no/such/file.ecproj\"}}",
|
||||
] {
|
||||
let r = replay(bad);
|
||||
assert!(!r.ok && !r.panicked, "{bad}: {r:?}");
|
||||
}
|
||||
let r = replay(&format!("{{\"command\": \"{}\"}}", "\u{1F600}".repeat(1000)));
|
||||
assert!(!r.ok && r.first_failure.and_then(|f| f.error).is_some_and(|e| e.ends_with('…')));
|
||||
assert!(replay("").ok, "an empty script is a new project");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_journal_exports_and_replays() {
|
||||
let mut s = Session::new();
|
||||
s.execute("comp.new", json!({"name": "Old"})).unwrap();
|
||||
s.execute("file.openDemoProject", json!({})).unwrap();
|
||||
s.execute("layer.newSolid", json!({"color": "#ff0000", "name": "Red"})).unwrap();
|
||||
s.execute("layer.newSolid", json!({"color": "#00ff00", "name": "Green"})).unwrap();
|
||||
// A query isn't journaled; neither is saving.
|
||||
s.execute("comp.info", json!({})).unwrap();
|
||||
let text = super::export(&s, None);
|
||||
let lines: Vec<&str> = text.lines().collect();
|
||||
assert_eq!(lines.len(), 4, "{text}");
|
||||
assert!(lines[0].contains("\"start\":\"new\"") && lines[1].contains("file.openDemoProject"), "starts at the last project open: {text}");
|
||||
let mut again = Session::new();
|
||||
let r = run(&mut again, &text, &Options::default());
|
||||
assert!(r.ok, "{r:?}");
|
||||
let layers = |s: &Session| s.active_comp().map(|c| c.layers.len());
|
||||
assert_eq!(layers(&again), layers(&s));
|
||||
// journal.export returns it; with `path` it writes the file.
|
||||
let v = s.execute("journal.export", json!({})).unwrap();
|
||||
assert_eq!(v["steps"], 3);
|
||||
assert_eq!(v["script"].as_str().unwrap().lines().skip(1).collect::<Vec<_>>(), lines[1..]);
|
||||
let dir = std::env::temp_dir().join(format!("ec-replay-{}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let path = dir.join("j.jsonl");
|
||||
s.execute("journal.export", json!({"path": path.to_str().unwrap()})).unwrap();
|
||||
assert!(std::fs::read_to_string(&path).unwrap().contains("layer.newSolid"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
assert_eq!(s.journal.iter().filter(|(id, _)| id.starts_with("journal.")).count(), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn help_export_repro_saves_into_the_settings_store() {
|
||||
let mut s = Session::new();
|
||||
assert!(s.execute("help.exportRepro", json!({})).is_err(), "no settings folder and no path");
|
||||
let store = Arc::new(MemoryConfig::default());
|
||||
s.config = Some(store.clone());
|
||||
s.execute("comp.new", json!({})).unwrap();
|
||||
let v = s.execute("help.exportRepro", json!({})).unwrap();
|
||||
assert_eq!(v["steps"], 1);
|
||||
let saved = store.list(super::REPRO_DIR);
|
||||
assert_eq!(saved.len(), 1, "{saved:?}");
|
||||
assert!(store.read(v["path"].as_str().unwrap()).unwrap().contains("comp.new"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_panicking_command_saves_a_crash_repro_and_replays_as_a_failure() {
|
||||
let mut s = Session::new();
|
||||
let store = Arc::new(MemoryConfig::default());
|
||||
s.config = Some(store.clone());
|
||||
s.execute("comp.new", json!({"name": "C"})).unwrap();
|
||||
// `test.panic` stands in for a command with a bug (see `commands::journal`).
|
||||
let e = s.execute("test.panic", json!({"why": "boom"})).unwrap_err().to_string();
|
||||
assert!(e.contains("boom") && e.contains("Repros/crash-"), "{e}");
|
||||
let names = store.list(super::REPRO_DIR);
|
||||
assert_eq!(names.len(), 1);
|
||||
let text = store.read(&format!("{}/{}", super::REPRO_DIR, names[0])).unwrap();
|
||||
assert!(text.lines().next().unwrap().contains("\"crash\""), "{text}");
|
||||
assert!(text.lines().last().unwrap().contains("test.panic"), "{text}");
|
||||
// The replay fails on the panic (and doesn't save another repro).
|
||||
let r = run(&mut Session::new(), &text, &Options::default());
|
||||
assert!(!r.ok && r.panicked, "{r:?}");
|
||||
assert_eq!(r.first_failure.as_ref().and_then(|f| f.command.as_deref()), Some("test.panic"));
|
||||
// A failing replay inside a session with a store saves nothing more.
|
||||
let mut t = Session::new();
|
||||
t.config = Some(store.clone());
|
||||
assert!(!run(&mut t, &text, &Options::default()).ok);
|
||||
assert_eq!(store.list(super::REPRO_DIR).len(), 1);
|
||||
// Only the newest crash repros are kept.
|
||||
for i in 0..30 {
|
||||
store.write(&format!("{}/crash-{i:012}-x.jsonl", super::REPRO_DIR), "{}").unwrap();
|
||||
}
|
||||
let _ = s.execute("test.panic", json!({}));
|
||||
assert_eq!(store.list(super::REPRO_DIR).len(), super::KEEP_CRASH_REPROS);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn commands_run_by_a_journaled_command_are_not_journaled_twice() {
|
||||
let mut s = Session::new();
|
||||
s.execute("comp.new", json!({})).unwrap();
|
||||
// engine.batch isn't journaled: its steps are.
|
||||
s.execute("engine.batch", json!({"steps": [{"command": "layer.newNull"}, {"command": "layer.newNull"}]})).unwrap();
|
||||
assert_eq!(s.journal.iter().filter(|(id, _)| id == "layer.newNull").count(), 2);
|
||||
// A journaled command's own steps replay with it.
|
||||
s.execute("file.runScript", json!({"steps": [{"command": "layer.newNull"}]})).unwrap();
|
||||
let ids: Vec<&str> = s.journal.iter().map(|(id, _)| id.as_str()).collect();
|
||||
assert_eq!(ids, ["comp.new", "layer.newNull", "layer.newNull", "file.runScript"]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
//! Regression tests as data: every `tests/repros/*.jsonl` (repository root) is a replay script
|
||||
//! (`effectcraft_engine::replay`, docs/agents.md "Repro scripts") run against a fully wired
|
||||
//! headless session, like `effectcraft-cli replay`. A crash fix's regression test is a `.jsonl`
|
||||
//! dropped there: it must replay without a panic and meet its expectations.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use effectcraft_engine::replay;
|
||||
|
||||
fn repros_dir() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../tests/repros")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_repro_replays_cleanly() {
|
||||
let mut files: Vec<PathBuf> =
|
||||
std::fs::read_dir(repros_dir()).unwrap().map(|e| e.unwrap().path()).filter(|p| p.extension().is_some_and(|x| x == "jsonl")).collect();
|
||||
files.sort();
|
||||
assert!(!files.is_empty(), "no repros in {}", repros_dir().display());
|
||||
let mut failures = vec![];
|
||||
for f in &files {
|
||||
let script = std::fs::read_to_string(f).unwrap();
|
||||
let mut s = effectcraft_host::session();
|
||||
let opts = replay::Options { keep_going: false, base_dir: f.parent().map(Path::to_path_buf) };
|
||||
let r = replay::run(&mut s, &script, &opts);
|
||||
if !r.ok {
|
||||
failures.push(format!("{}: {}", f.display(), serde_json::to_string(&r.first_failure).unwrap()));
|
||||
}
|
||||
}
|
||||
assert!(failures.is_empty(), "{} of {} repros failed:\n{}", failures.len(), files.len(), failures.join("\n"));
|
||||
}
|
||||
@@ -646,6 +646,68 @@ not read or written.
|
||||
`<comp>` is an id or name, or `-` for the active comp. `<value>` is JSON (`50`, `[960,540]`,
|
||||
`"#ff0000"`) or a bare string.
|
||||
|
||||
## Repro scripts
|
||||
|
||||
Every command the app runs (menus, panels, shortcuts, agents, scripts) goes into the session's
|
||||
command journal. A **repro script** is that journal as JSON lines, so a bug or crash becomes
|
||||
something an agent can replay and a regression test becomes a data file.
|
||||
|
||||
```text
|
||||
// comments: lines starting with // or #; blank lines are skipped
|
||||
{"start": "new" | "demo" | {"project": "rel/or/abs.ecproj"}, "allow_errors": false} optional, first line
|
||||
{"command": "comp.new", "params": {"name": "Main", "frameRate": 30}}
|
||||
{"command": "comp.settings", "params": {"frameRate": 0}, "expect_error": "frameRate"}
|
||||
{"expect": {"query": "comp.info", "pointer": "/frameRate", "equals": 30}}
|
||||
{"expect_no_error": true}
|
||||
```
|
||||
|
||||
- `start`: the project to begin with (default `new`). A relative `project` path is relative to the
|
||||
script. Other header keys (`version`, `app`, `os`, `crash`, `note`) are for people.
|
||||
- `command`: run like the app runs it. It must succeed, unless it has `expect_error` (`true`, or a
|
||||
text the error must contain) or the header says `allow_errors` (errors are reported but
|
||||
tolerated). A panic always fails the replay.
|
||||
- `expect`: run a command, normally a query (`prop.get`, `comp.info`, `layer.tree`, `project.summary`),
|
||||
and compare the value at the JSON `pointer` of its result with `equals` (numbers within 1e-6).
|
||||
Without `equals` it only checks that the query succeeds.
|
||||
- `expect_no_error`: the previous command succeeded.
|
||||
|
||||
Where scripts come from:
|
||||
|
||||
- `journal.export {path?}` returns `{script, steps}` (or writes `path`): the commands since the
|
||||
project was last replaced (`file.newProject`, `file.open`, `file.openDemoProject`). Queries and
|
||||
saves are not in it; commands a script or `file.runScript` ran are covered by that one step.
|
||||
- `help.exportRepro {path?}` saves the same for a bug report, by default to `Repros/` in the
|
||||
settings folder (macOS `~/Library/Application Support/EffectCraft/Repros`, next to `Logs/`).
|
||||
- **Crash repros**: when the last-resort guard catches a panic in a command, the app writes
|
||||
`Repros/crash-<unix time>-<command>.jsonl` there automatically (the newest 20 are kept): the
|
||||
journal, then the command that panicked, with `"allow_errors": true` and the error in the header.
|
||||
The error the user or agent sees names the file. Headless sessions without a settings folder
|
||||
(the CLI, tests) don't write one.
|
||||
|
||||
Privacy: a repro holds command ids and parameters only, never project, image or media file
|
||||
contents. Parameters can include file paths, layer and item names and text typed into text
|
||||
layers, so read a repro before sharing it.
|
||||
|
||||
Replay one headless (a fully wired session, like the CLI's other subcommands):
|
||||
|
||||
```sh
|
||||
effectcraft-cli replay crash.jsonl # per-step ok/FAIL lines, then the first failure
|
||||
effectcraft-cli replay crash.jsonl --json # {ok, panicked, steps, first_failure, skipped}
|
||||
effectcraft-cli replay crash.jsonl --keep-going
|
||||
```
|
||||
|
||||
It exits with status 1 when a step fails, an expectation is not met or a command panics, and
|
||||
stops at the first failure unless `--keep-going`. Steps that work on the user's files (`file.open`
|
||||
of their project, imports) need those files; replace them with a `start` project or generated
|
||||
content when you turn a report into a test. A repro only sees commands: UI state changed
|
||||
without one (a hover, a half-finished drag) is not in it.
|
||||
|
||||
**Regression tests are data files.** A crash fix's regression test is the repro, trimmed to the
|
||||
steps that matter, with an `expect_error` or `expect` line where the fix changed the outcome,
|
||||
dropped into [`tests/repros/`](../tests/repros). `crates/host/tests/repros.rs` replays every
|
||||
`*.jsonl` there and fails on any panic or unmet expectation
|
||||
(`cargo nextest run -p effectcraft-host --test repros`).
|
||||
|
||||
## Seeing the UI
|
||||
|
||||
To work on the UI, start the app with `cargo run -p effectcraft -- --control 9877` (add `--demo` to
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
// comp.new / comp.settings took a frameRate of 0 (or below 0.0005) as a 0/1 rate: the next
|
||||
// frame-duration computation divided by zero and killed the process with the unsaved project
|
||||
// (fixed in 780419c, "Compositions: reject zero and negative frame rates instead of panicking").
|
||||
{"start": "new"}
|
||||
{"command": "comp.new", "params": {"name": "Zero", "frameRate": 0}, "expect_error": "frameRate"}
|
||||
{"command": "comp.new", "params": {"name": "Tiny", "frameRate": 0.0001}, "expect_error": "frameRate"}
|
||||
{"command": "comp.new", "params": {"name": "Negative", "frameRate": -24}, "expect_error": "frameRate"}
|
||||
{"command": "comp.new", "params": {"name": "Main", "width": 320, "height": 180, "frameRate": 30, "duration": 1}}
|
||||
{"command": "comp.settings", "params": {"frameRate": 0}, "expect_error": "frameRate"}
|
||||
{"expect": {"query": "comp.info", "pointer": "/frameRate", "equals": 30}}
|
||||
{"command": "layer.newSolid", "params": {"color": "#ff0000"}}
|
||||
{"command": "time.set", "params": {"time": 0.5}}
|
||||
{"expect_no_error": true}
|
||||
@@ -0,0 +1,8 @@
|
||||
// Layer Settings' enablement read the first selected layer with `selected_layers[0]`: with layers
|
||||
// in the comp but none selected it indexed an empty list and panicked (fixed in b21e04b, M4.12).
|
||||
{"start": "demo"}
|
||||
{"command": "edit.deselectAll"}
|
||||
{"command": "layer.settings", "expect_error": "select a layer"}
|
||||
{"command": "layer.newSolid", "params": {"color": "#3366ff", "name": "Blue"}}
|
||||
{"command": "layer.settings", "params": {"name": "Renamed"}}
|
||||
{"expect_no_error": true}
|
||||
@@ -0,0 +1,17 @@
|
||||
// File > New Comp from Selection with Sequence Layers: a huge `overlapDuration` times the layer
|
||||
// count overflowed the duration arithmetic (a panic with overflow checks, a wrapped, wrong comp
|
||||
// length without them), and `dimensionsFrom` past the selection indexed out of bounds
|
||||
// (fixed in b21e04b, M4.12).
|
||||
{"start": "new"}
|
||||
{"command": "comp.new", "params": {"name": "Holder", "width": 64, "height": 64, "frameRate": 30, "duration": 2}}
|
||||
{"command": "layer.newSolid", "params": {"color": "#ff0000", "name": "a"}}
|
||||
{"command": "layer.newSolid", "params": {"color": "#00ff00", "name": "b"}}
|
||||
{"command": "layer.newSolid", "params": {"color": "#0000ff", "name": "c"}}
|
||||
{"command": "project.select", "params": {"items": ["a", "b", "c"]}}
|
||||
{"command": "file.newCompFromSelection", "params": {"single": true, "sequence": true, "overlap": true, "overlapDuration": 1e30, "duration": 2}}
|
||||
{"expect": {"query": "comp.info", "pointer": "/width", "equals": 64}}
|
||||
{"command": "project.select", "params": {"items": ["a", "b", "c"]}}
|
||||
{"command": "file.newCompFromSelection", "params": {"single": true, "sequence": true, "overlap": true, "overlapDuration": -5, "duration": 2}}
|
||||
{"expect": {"query": "comp.info", "pointer": "/duration", "equals": 4}}
|
||||
{"command": "project.select", "params": {"items": ["a", "b"]}}
|
||||
{"command": "file.newCompFromSelection", "params": {"single": true, "dimensionsFrom": 99}, "expect_error": "dimensionsFrom"}
|
||||
Reference in New Issue
Block a user