From bb78d55825adc879ae522c18ac57c4ce5690318d Mon Sep 17 00:00:00 2001 From: Valentin Klein Date: Wed, 4 Nov 2026 12:06:08 +0100 Subject: [PATCH] After Effects project import (.aep / .aepx); export follows (#2558) Add After Effects project import with bounded parsing, explicit loss reports, and regression coverage. Fetch the cleaned external test corpus at commit 97262276f625afdeeb61787e4cbb74b3f17af510 and verify each fixture against the pinned SHA-256 manifest. --- .gitignore | 4 + ATTRIBUTION.md | 2 + Cargo.lock | 16 + Cargo.toml | 1 + crates/aep/Cargo.toml | 27 + crates/aep/README.md | 89 ++ crates/aep/examples/aepdump.rs | 75 ++ crates/aep/examples/stubicc.rs | 359 ++++++ crates/aep/src/container.rs | 57 + crates/aep/src/container/aepx.rs | 640 ++++++++++ crates/aep/src/container/arbitrary.rs | 309 +++++ crates/aep/src/container/bytes.rs | 50 + crates/aep/src/container/camera.rs | 135 +++ crates/aep/src/container/chunk.rs | 213 ++++ crates/aep/src/container/effect_map.rs | 1038 +++++++++++++++++ crates/aep/src/container/effects.rs | 659 +++++++++++ crates/aep/src/container/error.rs | 57 + crates/aep/src/container/footage.rs | 332 ++++++ crates/aep/src/container/header.rs | 549 +++++++++ crates/aep/src/container/keys.rs | 361 ++++++ crates/aep/src/container/masks.rs | 258 ++++ crates/aep/src/container/matte.rs | 68 ++ crates/aep/src/container/read.rs | 142 +++ crates/aep/src/container/settings.rs | 224 ++++ crates/aep/src/container/shapes.rs | 273 +++++ crates/aep/src/container/style_map.rs | 368 ++++++ crates/aep/src/container/textdoc.rs | 1031 ++++++++++++++++ crates/aep/src/container/typed.rs | 539 +++++++++ crates/aep/src/container/values.rs | 263 +++++ crates/aep/src/container/write.rs | 86 ++ crates/aep/src/import.rs | 153 +++ crates/aep/src/import/build.rs | 906 ++++++++++++++ crates/aep/src/import/camera.rs | 114 ++ crates/aep/src/import/effects.rs | 552 +++++++++ crates/aep/src/import/markers.rs | 102 ++ crates/aep/src/import/masks.rs | 285 +++++ crates/aep/src/import/props.rs | 440 +++++++ crates/aep/src/import/report.rs | 241 ++++ crates/aep/src/import/settings.rs | 96 ++ crates/aep/src/import/shapes.rs | 492 ++++++++ crates/aep/src/import/styles.rs | 180 +++ crates/aep/src/import/text.rs | 453 +++++++ crates/aep/src/import/text_anim.rs | 202 ++++ crates/aep/src/import/time.rs | 112 ++ crates/aep/src/lib.rs | 24 + crates/aep/tests/fixtures/CASES.md | 390 +++++++ crates/aep/tests/fixtures/corpus.lock | 31 + crates/aep/tests/fixtures/synthetic.aep | Bin 0 -> 115806 bytes .../tests/fixtures/synthetic.aep.attribution | 7 + crates/aep/tests/fixtures/tools/3d.jsx | 44 + crates/aep/tests/fixtures/tools/README.md | 101 ++ crates/aep/tests/fixtures/tools/effects.jsx | 353 ++++++ .../aep/tests/fixtures/tools/fam-common.jsx | 37 + crates/aep/tests/fixtures/tools/fam-to-mcp.js | 78 ++ .../tests/fixtures/tools/footage-precomp.jsx | 44 + crates/aep/tests/fixtures/tools/keyframes.jsx | 107 ++ .../fixtures/tools/layers-comps-scrub-aepx.py | 25 + .../aep/tests/fixtures/tools/layers-comps.jsx | 173 +++ crates/aep/tests/fixtures/tools/markers.jsx | 21 + crates/aep/tests/fixtures/tools/masks.jsx | 75 ++ crates/aep/tests/fixtures/tools/mss-to-mcp.js | 63 + crates/aep/tests/fixtures/tools/patch-text.py | 72 ++ crates/aep/tests/fixtures/tools/scrub.py | 37 + crates/aep/tests/fixtures/tools/settings.jsx | 44 + crates/aep/tests/fixtures/tools/shapes.jsx | 96 ++ crates/aep/tests/fixtures/tools/styles.jsx | 84 ++ crates/aep/tests/fixtures/tools/text.jsx | 322 +++++ crates/aep/tests/fixtures/tools/to-mcp.js | 99 ++ crates/aep/tests/it/common/cases.rs | 381 ++++++ crates/aep/tests/it/common/mod.rs | 333 ++++++ crates/aep/tests/it/container.rs | 29 + crates/aep/tests/it/container/aepx.rs | 174 +++ crates/aep/tests/it/container/batch3_core.rs | 383 ++++++ .../aep/tests/it/container/batch3_objects.rs | 543 +++++++++ crates/aep/tests/it/container/batch5.rs | 152 +++ crates/aep/tests/it/container/cases_600.rs | 186 +++ crates/aep/tests/it/container/ease_round2.rs | 69 ++ crates/aep/tests/it/container/effects.rs | 292 +++++ .../aep/tests/it/container/effects_batch4.rs | 307 +++++ .../aep/tests/it/container/effects_batch5.rs | 151 +++ .../tests/it/container/effects_exposure.rs | 145 +++ crates/aep/tests/it/container/header.rs | 364 ++++++ crates/aep/tests/it/container/helpers.rs | 413 +++++++ crates/aep/tests/it/container/keyframes.rs | 230 ++++ .../aep/tests/it/container/layer_switches.rs | 225 ++++ crates/aep/tests/it/container/masks.rs | 203 ++++ crates/aep/tests/it/container/matte.rs | 82 ++ crates/aep/tests/it/container/odd_cases.rs | 266 +++++ crates/aep/tests/it/container/robustness.rs | 729 ++++++++++++ crates/aep/tests/it/container/roundtrip.rs | 130 +++ crates/aep/tests/it/container/shapes.rs | 128 ++ crates/aep/tests/it/container/text_fields.rs | 209 ++++ .../it/container/text_markers_footage.rs | 275 +++++ .../tests/it/container/text_style_slots.rs | 153 +++ crates/aep/tests/it/container/typed.rs | 268 +++++ crates/aep/tests/it/container/values.rs | 252 ++++ crates/aep/tests/it/import.rs | 27 + crates/aep/tests/it/import/aepx.rs | 95 ++ crates/aep/tests/it/import/blend_modes.rs | 87 ++ crates/aep/tests/it/import/cameras.rs | 92 ++ crates/aep/tests/it/import/cases.rs | 736 ++++++++++++ crates/aep/tests/it/import/combined.rs | 244 ++++ crates/aep/tests/it/import/ease_round2.rs | 80 ++ crates/aep/tests/it/import/effects.rs | 312 +++++ crates/aep/tests/it/import/effects_batch5.rs | 196 ++++ crates/aep/tests/it/import/effects_round2.rs | 139 +++ crates/aep/tests/it/import/effects_round3.rs | 323 +++++ crates/aep/tests/it/import/helpers.rs | 69 ++ crates/aep/tests/it/import/markers.rs | 96 ++ crates/aep/tests/it/import/masks.rs | 285 +++++ crates/aep/tests/it/import/mattes.rs | 105 ++ .../aep/tests/it/import/resaved_defaults.rs | 74 ++ crates/aep/tests/it/import/robustness.rs | 811 +++++++++++++ crates/aep/tests/it/import/shapes.rs | 203 ++++ crates/aep/tests/it/import/shapes_round2.rs | 114 ++ crates/aep/tests/it/import/styles.rs | 192 +++ crates/aep/tests/it/import/synthetic.rs | 297 +++++ crates/aep/tests/it/import/text.rs | 511 ++++++++ crates/aep/tests/it/import/text_anim.rs | 227 ++++ crates/aep/tests/it/import/timing.rs | 88 ++ crates/aep/tests/it/main.rs | 8 + crates/automation/src/confine.rs | 1 + crates/engine/Cargo.toml | 1 + crates/engine/src/cmd.rs | 2 + crates/engine/src/cmd/aep.rs | 101 ++ crates/engine/src/footage/doc_path.rs | 15 +- crates/engine/src/menus.rs | 1 + crates/engine/src/tests.rs | 1 + crates/engine/src/tests/aep_import.rs | 321 +++++ crates/project/src/item.rs | 5 + crates/project/src/lib.rs | 1 + crates/ui-egui/src/i18n.rs | 35 + crates/ui-egui/src/i18n/ui.rs | 58 + crates/ui-egui/src/lib.rs | 3 + crates/ui-egui/src/menus/frontend.rs | 10 + crates/ui-egui/src/menus/invoke.rs | 5 + crates/ui-egui/src/panels.rs | 1 + crates/ui-egui/src/panels/aep_losses.rs | 189 +++ crates/ui-egui/src/panels/dialogs.rs | 2 + docs/aep-format.md | 883 ++++++++++++++ docs/aep-import.md | 393 +++++++ docs/agents.md | 2 + docs/architecture.md | 1 + docs/file-format-parity.md | 12 +- docs/testing.md | 16 +- xtask/Cargo.toml | 1 + xtask/src/aep_corpus.rs | 290 +++++ xtask/src/main.rs | 11 +- 148 files changed, 29712 insertions(+), 12 deletions(-) create mode 100644 crates/aep/Cargo.toml create mode 100644 crates/aep/README.md create mode 100644 crates/aep/examples/aepdump.rs create mode 100644 crates/aep/examples/stubicc.rs create mode 100644 crates/aep/src/container.rs create mode 100644 crates/aep/src/container/aepx.rs create mode 100644 crates/aep/src/container/arbitrary.rs create mode 100644 crates/aep/src/container/bytes.rs create mode 100644 crates/aep/src/container/camera.rs create mode 100644 crates/aep/src/container/chunk.rs create mode 100644 crates/aep/src/container/effect_map.rs create mode 100644 crates/aep/src/container/effects.rs create mode 100644 crates/aep/src/container/error.rs create mode 100644 crates/aep/src/container/footage.rs create mode 100644 crates/aep/src/container/header.rs create mode 100644 crates/aep/src/container/keys.rs create mode 100644 crates/aep/src/container/masks.rs create mode 100644 crates/aep/src/container/matte.rs create mode 100644 crates/aep/src/container/read.rs create mode 100644 crates/aep/src/container/settings.rs create mode 100644 crates/aep/src/container/shapes.rs create mode 100644 crates/aep/src/container/style_map.rs create mode 100644 crates/aep/src/container/textdoc.rs create mode 100644 crates/aep/src/container/typed.rs create mode 100644 crates/aep/src/container/values.rs create mode 100644 crates/aep/src/container/write.rs create mode 100644 crates/aep/src/import.rs create mode 100644 crates/aep/src/import/build.rs create mode 100644 crates/aep/src/import/camera.rs create mode 100644 crates/aep/src/import/effects.rs create mode 100644 crates/aep/src/import/markers.rs create mode 100644 crates/aep/src/import/masks.rs create mode 100644 crates/aep/src/import/props.rs create mode 100644 crates/aep/src/import/report.rs create mode 100644 crates/aep/src/import/settings.rs create mode 100644 crates/aep/src/import/shapes.rs create mode 100644 crates/aep/src/import/styles.rs create mode 100644 crates/aep/src/import/text.rs create mode 100644 crates/aep/src/import/text_anim.rs create mode 100644 crates/aep/src/import/time.rs create mode 100644 crates/aep/src/lib.rs create mode 100644 crates/aep/tests/fixtures/CASES.md create mode 100644 crates/aep/tests/fixtures/corpus.lock create mode 100644 crates/aep/tests/fixtures/synthetic.aep create mode 100644 crates/aep/tests/fixtures/synthetic.aep.attribution create mode 100644 crates/aep/tests/fixtures/tools/3d.jsx create mode 100644 crates/aep/tests/fixtures/tools/README.md create mode 100644 crates/aep/tests/fixtures/tools/effects.jsx create mode 100644 crates/aep/tests/fixtures/tools/fam-common.jsx create mode 100644 crates/aep/tests/fixtures/tools/fam-to-mcp.js create mode 100644 crates/aep/tests/fixtures/tools/footage-precomp.jsx create mode 100644 crates/aep/tests/fixtures/tools/keyframes.jsx create mode 100644 crates/aep/tests/fixtures/tools/layers-comps-scrub-aepx.py create mode 100644 crates/aep/tests/fixtures/tools/layers-comps.jsx create mode 100644 crates/aep/tests/fixtures/tools/markers.jsx create mode 100644 crates/aep/tests/fixtures/tools/masks.jsx create mode 100644 crates/aep/tests/fixtures/tools/mss-to-mcp.js create mode 100644 crates/aep/tests/fixtures/tools/patch-text.py create mode 100644 crates/aep/tests/fixtures/tools/scrub.py create mode 100644 crates/aep/tests/fixtures/tools/settings.jsx create mode 100644 crates/aep/tests/fixtures/tools/shapes.jsx create mode 100644 crates/aep/tests/fixtures/tools/styles.jsx create mode 100644 crates/aep/tests/fixtures/tools/text.jsx create mode 100644 crates/aep/tests/fixtures/tools/to-mcp.js create mode 100644 crates/aep/tests/it/common/cases.rs create mode 100644 crates/aep/tests/it/common/mod.rs create mode 100644 crates/aep/tests/it/container.rs create mode 100644 crates/aep/tests/it/container/aepx.rs create mode 100644 crates/aep/tests/it/container/batch3_core.rs create mode 100644 crates/aep/tests/it/container/batch3_objects.rs create mode 100644 crates/aep/tests/it/container/batch5.rs create mode 100644 crates/aep/tests/it/container/cases_600.rs create mode 100644 crates/aep/tests/it/container/ease_round2.rs create mode 100644 crates/aep/tests/it/container/effects.rs create mode 100644 crates/aep/tests/it/container/effects_batch4.rs create mode 100644 crates/aep/tests/it/container/effects_batch5.rs create mode 100644 crates/aep/tests/it/container/effects_exposure.rs create mode 100644 crates/aep/tests/it/container/header.rs create mode 100644 crates/aep/tests/it/container/helpers.rs create mode 100644 crates/aep/tests/it/container/keyframes.rs create mode 100644 crates/aep/tests/it/container/layer_switches.rs create mode 100644 crates/aep/tests/it/container/masks.rs create mode 100644 crates/aep/tests/it/container/matte.rs create mode 100644 crates/aep/tests/it/container/odd_cases.rs create mode 100644 crates/aep/tests/it/container/robustness.rs create mode 100644 crates/aep/tests/it/container/roundtrip.rs create mode 100644 crates/aep/tests/it/container/shapes.rs create mode 100644 crates/aep/tests/it/container/text_fields.rs create mode 100644 crates/aep/tests/it/container/text_markers_footage.rs create mode 100644 crates/aep/tests/it/container/text_style_slots.rs create mode 100644 crates/aep/tests/it/container/typed.rs create mode 100644 crates/aep/tests/it/container/values.rs create mode 100644 crates/aep/tests/it/import.rs create mode 100644 crates/aep/tests/it/import/aepx.rs create mode 100644 crates/aep/tests/it/import/blend_modes.rs create mode 100644 crates/aep/tests/it/import/cameras.rs create mode 100644 crates/aep/tests/it/import/cases.rs create mode 100644 crates/aep/tests/it/import/combined.rs create mode 100644 crates/aep/tests/it/import/ease_round2.rs create mode 100644 crates/aep/tests/it/import/effects.rs create mode 100644 crates/aep/tests/it/import/effects_batch5.rs create mode 100644 crates/aep/tests/it/import/effects_round2.rs create mode 100644 crates/aep/tests/it/import/effects_round3.rs create mode 100644 crates/aep/tests/it/import/helpers.rs create mode 100644 crates/aep/tests/it/import/markers.rs create mode 100644 crates/aep/tests/it/import/masks.rs create mode 100644 crates/aep/tests/it/import/mattes.rs create mode 100644 crates/aep/tests/it/import/resaved_defaults.rs create mode 100644 crates/aep/tests/it/import/robustness.rs create mode 100644 crates/aep/tests/it/import/shapes.rs create mode 100644 crates/aep/tests/it/import/shapes_round2.rs create mode 100644 crates/aep/tests/it/import/styles.rs create mode 100644 crates/aep/tests/it/import/synthetic.rs create mode 100644 crates/aep/tests/it/import/text.rs create mode 100644 crates/aep/tests/it/import/text_anim.rs create mode 100644 crates/aep/tests/it/import/timing.rs create mode 100644 crates/aep/tests/it/main.rs create mode 100644 crates/engine/src/cmd/aep.rs create mode 100644 crates/engine/src/tests/aep_import.rs create mode 100644 crates/ui-egui/src/panels/aep_losses.rs create mode 100644 docs/aep-format.md create mode 100644 docs/aep-import.md create mode 100644 xtask/src/aep_corpus.rs diff --git a/.gitignore b/.gitignore index 4a79323c8..ff6ad2397 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,7 @@ target/ /log .DS_Store .claude/worktrees/ +__pycache__/ + +# After Effects test corpus fetched by `cargo xtask aep-corpus` +/corpus/ diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index c97d32ed9..93b34b9bf 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -64,6 +64,8 @@ and licence texts are in [`NOTICE`](NOTICE). | `crates/model/tests/fixtures/cube.obj` | EffectCraft contributors | Original work: Wavefront OBJ test fixture (unit cube, two materials) | MIT OR Apache-2.0 | | `crates/model/tests/fixtures/cube.mtl` | EffectCraft contributors | Original work: MTL test fixture (materials of cube.obj) | MIT OR Apache-2.0 | +One small After Effects test project of `crates/aep` is in this repository: `crates/aep/tests/fixtures/synthetic.aep` (original work, made with After Effects 2026 from own data, working space None, the embedded colour profile removed; MIT OR Apache-2.0, with its `.attribution` sidecar). The other After Effects test projects of `crates/aep` are not in this repository. They are original work of an EffectCraft contributor, generated with After Effects 2026 as own test data (MIT OR Apache-2.0, each file with its `.attribution` sidecar); they are in [storytold/effectcraft-corpus](https://github.com/storytold/effectcraft-corpus) with their sidecars, fetched by `cargo xtask aep-corpus` (commit-pinned, SHA-256-verified), and the tests read them only from the folder named by `EFFECTCRAFT_AEP_FIXTURES` (opt-in). Their embedded colour profiles are replaced by stubs made with `crates/aep/examples/stubicc.rs` (a header and a description, no colour data), so no vendor profile data is in them. + ## Screenshots Screenshots of EffectCraft itself, rendered headlessly, showing only procedurally generated demo content. diff --git a/Cargo.lock b/Cargo.lock index 232fa6b18..9a9f78ecd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1436,6 +1436,20 @@ dependencies = [ "zbus", ] +[[package]] +name = "effectcraft-aep" +version = "0.21.0" +dependencies = [ + "effectcraft-color", + "effectcraft-effects", + "effectcraft-keyframe", + "effectcraft-project", + "effectcraft-time", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "effectcraft-automation" version = "0.21.0" @@ -1507,6 +1521,7 @@ version = "0.21.0" dependencies = [ "chrono", "craft-docpath", + "effectcraft-aep", "effectcraft-color", "effectcraft-effects", "effectcraft-geom", @@ -7630,6 +7645,7 @@ dependencies = [ "chrono", "pkg-config", "serde_json", + "sha2", "tempfile", ] diff --git a/Cargo.toml b/Cargo.toml index f5f84775f..fa1870e88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,6 +40,7 @@ effectcraft-lottie = { path = "crates/lottie" } effectcraft-interchange = { path = "crates/interchange" } effectcraft-pdf = { path = "crates/pdf" } effectcraft-psd = { path = "crates/psd" } +effectcraft-aep = { path = "crates/aep" } effectcraft-vp9enc = { path = "crates/vp9enc" } effectcraft-svg = { path = "crates/svg" } effectcraft-engine = { path = "crates/engine" } diff --git a/crates/aep/Cargo.toml b/crates/aep/Cargo.toml new file mode 100644 index 000000000..c538dd4f5 --- /dev/null +++ b/crates/aep/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "effectcraft-aep" +description = "After Effects projects: the .aep (RIFX 'Egg!') / .aepx container as a lossless chunk tree with a typed view, and its import into an EffectCraft project, which reports everything it does not carry over" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +effectcraft-time = { workspace = true } +effectcraft-color = { workspace = true } +effectcraft-keyframe = { workspace = true } +effectcraft-project = { workspace = true } +effectcraft-effects = { workspace = true } +serde = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +serde_json = { workspace = true } + +# The tool's own tests run with the crate's (examples are only built by default). +[[example]] +name = "stubicc" +test = true diff --git a/crates/aep/README.md b/crates/aep/README.md new file mode 100644 index 000000000..a9b3b807d --- /dev/null +++ b/crates/aep/README.md @@ -0,0 +1,89 @@ +# effectcraft-aep + +After Effects projects (`.aep`, `.aepx`) in one crate, the way `effectcraft-lottie` handles Lottie: + +- `container`: reads and writes the container as a lossless chunk tree, with a typed, read-only + view (below; its API is re-exported at the crate root); +- `import`: maps the typed view into an EffectCraft project and reports everything it does not + import ([`docs/aep-import.md`](../../docs/aep-import.md)). + +Writing an EffectCraft project as an `.aep` (export) is not part of this crate yet. + +No file access: callers pass bytes, so the crate builds for wasm32. + +## Container + +An `.aep` is a `RIFX` container (big-endian RIFF) with form type `Egg!`: chunks of a 4-byte id, a +big-endian `u32` size and a body padded to an even length; `LIST` chunks start with a 4-byte list +type followed by chunks. `parse` builds the tree, `write` serialises it, and for every input that +parses, `write(parse(x)) == x` byte for byte (container and chunk sizes are recomputed from the +tree; pad bytes, odd bodies and the bytes after the container, an XMP packet, are kept). + +The XML form of a project (`.aepx`) is read into the same tree by `read_aepx` (a small bounded +XML tokenizer, no dependency); `parse_any` picks the reader by content, so the typed view and the +importer take either form. The mapping is in `docs/aep-format.md`, section 1. + +Bodies that look like lists but are not chunk sequences (for example `LIST btdk`, raw text) stay +opaque rather than failing the parse. Input is untrusted: sizes are checked against the remaining +input, nesting and chunk count are bounded (`Limits`), nothing is allocated from a declared size, +and truncated or damaged files return `Err` (or, inside the container, an opaque body). + +The container layer decodes no chunk body. On top of it, a typed view (`Project`, `Item`, `Comp`, `Layer`, +`Property`, ...) reads the fields whose layout was verified against the values the test files were made +with: layer and composition headers (switches, exact times and time stretch, size, frame rate, work area, shutter phase, display start, renderer), +static transform values in After Effects' units with the "absent = default" rule, keyframe records +(interpolation sides, ease per component, plain and spatial templates), Time Remapping, expressions with their enable switch, effect +instances and parameter values, masks and bezier paths with tangents, shape layer groups, text documents +(paragraph and style runs, box, justification, tracking, faux styles, caps, leading), markers with +duration, strings and cue data, cameras and lights with their options, footage (stills, audio, image +sequences), solids and precomp links. Every accessor returns `Option`/an empty list on damaged data and never +panics. Which fields are verified, which are not decoded, and how it was found is in +[`docs/aep-format.md`](../../docs/aep-format.md). + +## Specifications + +Public specifications used for parts the format embeds: + +- ICC.1:2010: the profile description of a working-space profile (a v2 `desc` or a v4 `mluc` + tag), from which the working space's name is read (`container/settings.rs`). +- RFC 4648 (base64): the profile data in the project's colour settings JSON. + +## Clean room + +Everything here comes from our own observations: project files that we generated ourselves with +After Effects, one change per file, compared byte by byte. No source code or format +documentation of another `.aep` reader or writer was used, and no Adobe asset is used. The test fixtures are such generated files +(own test data): a few combined projects, one composition per case (`tests/fixtures/CASES.md`; +the generator is in `tests/fixtures/tools/`). They are not in this repository but in +[storytold/effectcraft-corpus](https://github.com/storytold/effectcraft-corpus), with their +attribution, the generation notes and the inspection of their embedded content. +`cargo xtask aep-corpus` fetches them at the commit pinned in `tests/fixtures/corpus.lock`, checks +every file against the SHA-256 listed there and only then installs them in `corpus/aep` (ignored by +git). The tests read them from the folder named by the `EFFECTCRAFT_AEP_FIXTURES` environment +variable (opt-in; set `EFFECTCRAFT_AEP_FIXTURES_REQUIRED=1` too, so that a missing project fails). The +tests reach them only through `tests/it/common/mod.rs`, which hands a test its case as a project +of its own cut from the combined file; a test whose fixture is missing (or with the variable not +set) prints `SKIPPED: fixture missing` and passes. + +`tests/fixtures/synthetic.aep` (115 KB) is the one project in the repository: made in After Effects 2026 +with the working space set to None (one composition, an image sequence of three 8x8 PNGs, a +solid, a folder, a Levels and a Glow effect with expressions). Every After Effects project embeds +the working space's ICC profile (HP sRGB IEC61966-2.1 for the default) in a JSON settings chunk; +the one `Utf8` chunk of this file that held a profile was replaced by `{}`, and nothing else was +changed. The tests in `tests/it/import/synthetic.rs` damage copies of it and need no other file. + +## Tools + +- `tests/fixtures/tools/`: the generator of the test projects (regenerates the corpus with After + Effects; its README says how). +- `examples/aepdump.rs`: prints the chunk tree of a file (ids, sizes, short data, match names): + `cargo run -p effectcraft-aep --example aepdump -- file.aep`. +- `examples/stubicc.rs`: replaces every embedded ICC profile of a saved project by a stub that keeps + only the profile's description: the JSON `colorProfileData` of the working space (any white space + around the colon, `\/` escapes, quotes as `"`) in an `.aep` chunk or the `.aepx` text, and a raw + profile of an imported image (an `.aep` chunk, or the `bdata` hex of an `.aepx`): + `cargo run -p effectcraft-aep --example stubicc -- in.aep out.aep`. A profile it cannot replace + stops it before anything is written. The importer reads the working space's name from the + description only, so the project imports the same; no vendor colour data stays in the test + project. Check the result independently, not by the tool's count (the corpus repository has + `tools/aep/inspect-embedded.py`, which scans the raw bytes, base64 and hex). diff --git a/crates/aep/examples/aepdump.rs b/crates/aep/examples/aepdump.rs new file mode 100644 index 000000000..2fe492696 --- /dev/null +++ b/crates/aep/examples/aepdump.rs @@ -0,0 +1,75 @@ +//! Developer tool: prints the chunk tree of an .aep (ids, sizes, short data), match names inline. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +use effectcraft_aep::{Body, Chunk}; + +/// Ends this developer tool with a message instead of a panic. +trait OrExit { + fn or_exit(self, what: &str) -> T; +} + +impl OrExit for Option { + fn or_exit(self, what: &str) -> T { + match self { + Some(v) => v, + None => exit(what), + } + } +} + +impl OrExit for Result { + fn or_exit(self, what: &str) -> T { + match self { + Ok(v) => v, + Err(e) => exit(&format!("{what}: {e:?}")), + } + } +} + +fn exit(msg: &str) -> ! { + eprintln!("{msg}"); + std::process::exit(1) +} + +fn show(c: &Chunk<'_>, depth: usize, max: usize, hex: usize) { + let ind = " ".repeat(depth); + let id = String::from_utf8_lossy(&c.id).to_string(); + match &c.body { + Body::Data(d) if &c.id == b"tdmn" => { + let n: String = d.iter().take_while(|b| **b != 0).map(|b| *b as char).collect(); + println!("{ind}tdmn {n}"); + } + Body::Data(d) => { + let h: String = d.iter().take(hex).map(|b| format!("{b:02x}")).collect(); + let a: String = d.iter().take(hex).map(|b| if (32..127).contains(b) { *b as char } else { '.' }).collect(); + println!("{ind}{id} [{}] {h} |{a}|", d.len()); + } + Body::Chunks(k) => { + println!("{ind}{id} (nested)"); + if depth < max { + for x in k { + show(x, depth + 1, max, hex); + } + } + } + Body::List { list_type, children } => { + println!("{ind}LIST {} ({} children)", String::from_utf8_lossy(list_type), children.len()); + if depth < max { + for x in children { + show(x, depth + 1, max, hex); + } + } + } + Body::OpaqueList { list_type, data } => println!("{ind}LIST {} opaque [{}]", String::from_utf8_lossy(list_type), data.len()), + } +} +fn main() { + let a: Vec = std::env::args().collect(); + let bytes = std::fs::read(a.get(1).or_exit("usage: see the comment at the top of this file")).or_exit("read"); + let max: usize = a.get(2).and_then(|s| s.parse().ok()).unwrap_or(4); + let hex: usize = a.get(3).and_then(|s| s.parse().ok()).unwrap_or(32); + let f = effectcraft_aep::parse(&bytes).or_exit("parse"); + for c in &f.chunks { + show(c, 0, max, hex); + } + println!("trailer {} bytes", f.trailer.len()); +} diff --git a/crates/aep/examples/stubicc.rs b/crates/aep/examples/stubicc.rs new file mode 100644 index 000000000..9438da06d --- /dev/null +++ b/crates/aep/examples/stubicc.rs @@ -0,0 +1,359 @@ +//! Developer tool: replaces every embedded ICC profile of an .aep / .aepx by a stub that keeps only the +//! profile's description. Fixture hygiene: a saved project carries the vendor's working-space profile +//! (a few KB of colour data); the importer reads only the profile's name, the `desc` tag, so the stub +//! (a 128-byte header, one `desc` tag, no colour data, made here) imports identically. +//! +//! `cargo run -p effectcraft-aep --example stubicc -- in.aep out.aep` +//! +//! It handles: +//! - the JSON value `"colorProfileData": ""` in any chunk of an .aep and anywhere in the text +//! of an .aepx (white space around the colon, `\/` escapes and quotes written as `"` allowed); +//! - a raw ICC profile stored as a chunk of its own (the profile of an imported image), in an .aep +//! as the chunk's bytes and in an .aepx as its `bdata=""` attribute. +//! +//! A profile it cannot replace (no readable description, data that does not decode) stops the tool +//! before anything is written. Check the result independently (the corpus has +//! `tools/aep/inspect-embedded.py`), not by this tool's own count. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +use effectcraft_aep::{Body, Chunk}; + +const B64: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + +/// Ends this developer tool with a message instead of a panic. +trait OrExit { + fn or_exit(self, what: &str) -> T; +} + +impl OrExit for Option { + fn or_exit(self, what: &str) -> T { + match self { + Some(v) => v, + None => exit(what), + } + } +} + +impl OrExit for Result { + fn or_exit(self, what: &str) -> T { + match self { + Ok(v) => v, + Err(e) => exit(&format!("{what}: {e:?}")), + } + } +} + +fn exit(msg: &str) -> ! { + eprintln!("{msg}"); + std::process::exit(1) +} + +fn b64_decode(text: &[u8]) -> Option> { + let (mut out, mut acc, mut bits) = (Vec::new(), 0u32, 0u32); + for &c in text.iter().filter(|c| **c != b'=') { + let v = u32::try_from(B64.iter().position(|b| *b == c)?).ok()?; + acc = (acc << 6 | v) & 0x00ff_ffff; + bits += 6; + if bits >= 8 { + bits -= 8; + out.push(((acc >> bits) & 0xff) as u8); + } + } + Some(out) +} + +fn b64_encode(data: &[u8]) -> Vec { + let mut out = Vec::new(); + for ch in data.chunks(3) { + let at = |i: usize| u32::from(ch.get(i).copied().unwrap_or(0)); + let n = (at(0) << 16) | (at(1) << 8) | at(2); + for i in 0..4usize { + if i <= ch.len() { + out.push(B64.get(((n >> (18 - 6 * i)) & 63) as usize).copied().unwrap_or(b'A')); + } else { + out.push(b'='); + } + } + } + out +} + +fn be32(d: &[u8], at: usize) -> Option { + let b: [u8; 4] = d.get(at..at.checked_add(4)?)?.try_into().ok()?; + usize::try_from(u32::from_be_bytes(b)).ok() +} + +/// The profile description (ICC.1 `desc`: v2 ASCII or v4 `mluc`, first record). +fn description(icc: &[u8]) -> Option { + let count = be32(icc, 128)?; + let tag = (0..count.min(1024)).find_map(|i| { + let at = 132usize.checked_add(i.checked_mul(12)?)?; + (icc.get(at..at.checked_add(4)?)? == b"desc").then_some(at) + })?; + let off = be32(icc, tag.checked_add(4)?)?; + let body = icc.get(off..off.checked_add(be32(icc, tag.checked_add(8)?)?)?)?; + let text = match body.get(..4)? { + b"desc" => { + let n = be32(body, 8)?; + String::from_utf8_lossy(body.get(12..12usize.checked_add(n)?)?.split(|b| *b == 0).next()?).into_owned() + } + b"mluc" => { + let (n, at) = (be32(body, 20)?, be32(body, 24)?); + let units: Vec = body.get(at..at.checked_add(n)?)?.chunks(2).filter_map(|p| Some(u16::from_be_bytes([*p.first()?, *p.get(1)?]))).collect(); + String::from_utf16_lossy(&units) + } + _ => return None, + }; + Some(text.trim_end_matches('\0').trim().to_owned()) +} + +/// A profile with a header, one `desc` tag and no colour data. +fn stub(desc: &str) -> Option> { + let mut body = b"desc\0\0\0\0".to_vec(); + body.extend_from_slice(&u32::try_from(desc.len().checked_add(1)?).ok()?.to_be_bytes()); + body.extend_from_slice(desc.as_bytes()); + body.push(0); + body.extend_from_slice(&[0; 4 + 4 + 2 + 1 + 67]); + let mut p = vec![0u8; 128]; + for (at, v) in [(8, &0x0240_0000u32.to_be_bytes()[..]), (12, b"mntr"), (16, b"RGB "), (20, b"XYZ "), (36, b"acsp")] { + p.get_mut(at..at + v.len())?.copy_from_slice(v); + } + p.extend_from_slice(&1u32.to_be_bytes()); + p.extend_from_slice(b"desc"); + p.extend_from_slice(&144u32.to_be_bytes()); + p.extend_from_slice(&u32::try_from(body.len()).ok()?.to_be_bytes()); + p.extend_from_slice(&body); + let size = u32::try_from(p.len()).ok()?.to_be_bytes(); + p.get_mut(..4)?.copy_from_slice(&size); + Some(p) +} + +/// The JSON key, without its quotes. +const KEY: &[u8] = b"colorProfileData"; +/// How a JSON quote is written: as is (a `Utf8` chunk, the `` text of an .aepx) and as an +/// XML entity (JSON inside an attribute of an .aepx). +const QUOTES: [&[u8]; 2] = [b"\"", b"""]; + +/// What the tool did, counted. +#[derive(Default)] +struct Tally { + stubbed: usize, + /// Values too short to be a profile (no ICC header), kept as they are. + kept_short: usize, + /// Profiles that could not be replaced (no readable description, undecodable data): the tool + /// fails, so no profile is ever left behind silently. + failed: Vec, +} + +fn is_profile(d: &[u8]) -> bool { + d.len() > 132 && d.get(36..40) == Some(b"acsp") +} + +/// The stub for the bytes `d` of a value, or `None` (counted) when it holds no profile or the +/// profile has no readable description. +fn stub_of(d: &[u8], at: usize, tally: &mut Tally) -> Option> { + if !is_profile(d) { + tally.kept_short += 1; + return None; + } + let s = description(d).and_then(|t| stub(&t)); + if s.is_none() { + tally.failed.push(format!("profile at byte {at}: no readable description")); + } + s +} + +/// At the start of `t`: the key in quotes, optional white space, a colon, optional white space and +/// the opening quote of the value. Returns the length up to and including that quote, and the quote. +fn value_start(t: &[u8]) -> Option<(usize, &'static [u8])> { + let ws = |mut i: usize| { + while t.get(i).is_some_and(u8::is_ascii_whitespace) { + i += 1; + } + i + }; + QUOTES.into_iter().find_map(|q| { + let key_end = q.len().checked_mul(2)?.checked_add(KEY.len())?; + let quoted = t.get(..key_end)?; + if !(quoted.starts_with(q) && quoted.get(q.len()..q.len() + KEY.len())? == KEY && quoted.ends_with(q)) { + return None; + } + let colon = ws(key_end); + if t.get(colon) != Some(&b':') { + return None; + } + let open = ws(colon + 1); + t.get(open..)?.starts_with(q).then_some((open + q.len(), q)) + }) +} + +fn hex_decode(h: &[u8]) -> Option> { + if !h.len().is_multiple_of(2) { + return None; + } + h.chunks(2).map(|p| u8::from_str_radix(std::str::from_utf8(p).ok()?, 16).ok()).collect() +} + +fn hex_encode(d: &[u8], upper: bool) -> Vec { + d.iter().flat_map(|b| if upper { format!("{b:02X}") } else { format!("{b:02x}") }.into_bytes()).collect() +} + +/// Replaces, in `text`, every `colorProfileData` JSON value (base64; `\/` escapes, white space +/// around the colon and quotes written as `"` allowed) and every .aepx `bdata=""` +/// attribute that holds a raw profile. +fn stub_text(text: &[u8], tally: &mut Tally) -> Vec { + const BDATA: &[u8] = b"bdata=\""; + let (mut out, mut i) = (Vec::with_capacity(text.len()), 0); + while let Some(&b) = text.get(i) { + let rest = text.get(i..).unwrap_or_default(); + if let Some((open, q)) = value_start(rest) + && let Some(len) = rest.get(open..).and_then(|v| v.windows(q.len()).position(|w| w == q)) + { + let raw = rest.get(open..open + len).unwrap_or_default(); + let unescaped: Vec = raw.iter().copied().filter(|c| *c != b'\\').collect(); + let replaced = match b64_decode(&unescaped) { + Some(d) => stub_of(&d, i, tally).map(|s| b64_encode(&s)), + None => { + tally.failed.push(format!("colorProfileData at byte {i}: not base64")); + None + } + }; + out.extend_from_slice(rest.get(..open).unwrap_or_default()); + match replaced { + Some(enc) => { + out.extend_from_slice(&enc); + tally.stubbed += 1; + } + None => out.extend_from_slice(raw), + } + i += open + len; + continue; + } + if rest.starts_with(BDATA) + && let Some(len) = rest.get(BDATA.len()..).and_then(|v| v.iter().position(|c| *c == b'"')) + { + let hex = rest.get(BDATA.len()..BDATA.len() + len).unwrap_or_default(); + let replaced = hex_decode(hex).filter(|d| is_profile(d)).and_then(|d| stub_of(&d, i, tally)); + out.extend_from_slice(BDATA); + match replaced { + Some(s) => { + out.extend_from_slice(&hex_encode(&s, hex.iter().any(u8::is_ascii_uppercase))); + tally.stubbed += 1; + } + None => out.extend_from_slice(hex), + } + i += BDATA.len() + len; + continue; + } + out.push(b); + i += 1; + } + out +} + +fn walk(chunks: &mut [Chunk<'_>], tally: &mut Tally) { + for c in chunks.iter_mut() { + // Only a data chunk is rewritten as one: an opaque `LIST` also has bytes, but replacing it by + // a data chunk would drop its list type. + let data = if let Body::Data(d) = &c.body { Some(d.to_vec()) } else { None }; + match data { + Some(d) if d.windows(KEY.len()).any(|w| w == KEY) => { + let new = stub_text(&d, tally); + *c = Chunk::data(c.id, new); + } + Some(d) if is_profile(&d) => { + if let Some(s) = stub_of(&d, 0, tally) { + *c = Chunk::data(c.id, s); + tally.stubbed += 1; + } + } + _ => { + if let Body::List { children, .. } | Body::Chunks(children) = &mut c.body { + walk(children, tally); + } + } + } + } +} + +fn main() { + let args: Vec = std::env::args().collect(); + let (Some(input), Some(output)) = (args.get(1), args.get(2)) else { exit("usage: stubicc ") }; + let bytes = std::fs::read(input).or_exit("read the input"); + let mut tally = Tally::default(); + let out = if input.ends_with(".aepx") { + stub_text(&bytes, &mut tally) + } else { + let mut file = effectcraft_aep::parse(&bytes).or_exit("parse the input"); + walk(&mut file.chunks, &mut tally); + effectcraft_aep::write(&file).or_exit("write the output") + }; + if !tally.failed.is_empty() { + exit(&format!("nothing written: {} profile(s) could not be replaced:\n{}", tally.failed.len(), tally.failed.join("\n"))); + } + std::fs::write(output, out).or_exit("write the output"); + println!("stubbed {} profile(s); kept {} short value(s) that hold no profile", tally.stubbed, tally.kept_short); +} + +#[cfg(test)] +#[allow(clippy::unwrap_used)] +mod tests { + use super::*; + + fn profile(desc: &str) -> Vec { + // A profile with colour data after the description (like a real one): the stub of `desc` + // plus 3000 bytes, its size field updated. + let mut p = stub(desc).unwrap(); + p.extend(std::iter::repeat_n(0x5a, 3000)); + let size = u32::try_from(p.len()).unwrap().to_be_bytes(); + p[..4].copy_from_slice(&size); + p + } + + fn stubbed(text: &[u8]) -> (Vec, Tally) { + let mut tally = Tally::default(); + (stub_text(text, &mut tally), tally) + } + + #[test] + fn json_values_in_every_spelling_are_replaced() { + let b64 = String::from_utf8(b64_encode(&profile("sRGB IEC61966-2.1"))).unwrap(); + let want = String::from_utf8(b64_encode(&stub("sRGB IEC61966-2.1").unwrap())).unwrap(); + for (before, after) in [ + (format!("{{\"colorProfileData\":\"{b64}\"}}"), format!("{{\"colorProfileData\":\"{want}\"}}")), + (format!("{{\"colorProfileData\" : \n\t\"{b64}\"}}"), format!("{{\"colorProfileData\" : \n\t\"{want}\"}}")), + (format!("{{\"colorProfileData\":\"{}\"}}", b64.replace('/', "\\/")), format!("{{\"colorProfileData\":\"{want}\"}}")), + (format!("{{"colorProfileData": "{b64}"}}"), format!("{{"colorProfileData": "{want}"}}")), + ] { + let (out, tally) = stubbed(before.as_bytes()); + assert_eq!(String::from_utf8(out).unwrap(), after); + assert_eq!((tally.stubbed, tally.failed.len()), (1, 0)); + } + } + + #[test] + fn a_raw_profile_in_aepx_hex_is_replaced_and_other_hex_kept() { + let hex = String::from_utf8(hex_encode(&profile("Rec.709 Gamma 2.4"), false)).unwrap(); + let want = String::from_utf8(hex_encode(&stub("Rec.709 Gamma 2.4").unwrap(), false)).unwrap(); + let text = format!("", "ab".repeat(200)); + let (out, tally) = stubbed(text.as_bytes()); + assert_eq!(String::from_utf8(out).unwrap(), format!("", "ab".repeat(200))); + assert_eq!((tally.stubbed, tally.failed.len()), (1, 0)); + // Upper-case hex stays upper case. + let (out, _) = stubbed(format!("bdata=\"{}\"", hex.to_uppercase()).as_bytes()); + assert_eq!(String::from_utf8(out).unwrap(), format!("bdata=\"{}\"", want.to_uppercase())); + } + + #[test] + fn a_profile_that_cannot_be_replaced_is_a_failure_not_a_silent_copy() { + // An ICC header whose tag table has no `desc`. + let mut p = profile("x"); + p[132..136].copy_from_slice(b"wtpt"); + let (_, tally) = stubbed(format!("\"colorProfileData\":\"{}\"", String::from_utf8(b64_encode(&p)).unwrap()).as_bytes()); + assert_eq!((tally.stubbed, tally.failed.len()), (0, 1)); + let (_, tally) = stubbed(b"\"colorProfileData\":\"not*base64\""); + assert_eq!(tally.failed.len(), 1); + // A short value holds no profile: kept and counted, not a failure. + let (out, tally) = stubbed(b"\"colorProfileData\":\"AAAA\""); + assert_eq!((out.as_slice(), tally.kept_short, tally.failed.len()), (&b"\"colorProfileData\":\"AAAA\""[..], 1, 0)); + } +} diff --git a/crates/aep/src/container.rs b/crates/aep/src/container.rs new file mode 100644 index 000000000..4aa0f393a --- /dev/null +++ b/crates/aep/src/container.rs @@ -0,0 +1,57 @@ +//! After Effects project (`.aep`) container reader and writer. +//! +//! An `.aep` is a `RIFX` container (big-endian RIFF) with form type `Egg!`. This crate reads it +//! into a lossless chunk tree ([`AepFile`]) and writes it back byte for byte. The XML form of a +//! project (`.aepx`) is read into the same tree by [`read_aepx`]; [`parse_any`] picks the reader +//! by content. See +//! `docs/aep-format.md` for the observed format and how it was established (clean room: only +//! files we generated ourselves, one change per file). +//! +//! [`Project::new`] opens a typed, read-only view over the tree: items, compositions +//! ([`CompHeader`]), layers ([`LayerHeader`], [`Transform`], effects, masks, text documents, +//! markers), properties, keyframes and footage. Only fields whose layout was verified against the +//! values the test files were made with are decoded; every accessor returns `None` (or an empty +//! list) on damaged data instead of failing. + +mod aepx; +mod arbitrary; +mod bytes; +mod camera; +mod chunk; +pub mod effect_map; +mod effects; +mod error; +mod footage; +mod header; +mod keys; +mod masks; +mod matte; +mod read; +mod settings; +mod shapes; +pub mod style_map; +mod textdoc; +mod typed; +mod values; +mod write; + +pub use aepx::{is_aepx, parse_any, read_aepx, read_aepx_with_limits}; +pub use arbitrary::{CURVE_IDENTITY, CurvesData, HueSaturationData, LevelsData, MeshData, curve_lut}; +pub use camera::{CameraPosition, DEFAULT_CAMERA_ZOOM}; +pub use chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS, Walk}; +pub use effects::{DefinitionSource, Effect, EffectDefinitions, EffectParam, LayerRef, ParamKind, ParamValue}; +pub use error::AepError; +pub use footage::{Footage, SequenceInfo}; +pub use header::{BlendMode, CompHeader, LayerHeader, LayerType, LightType, Quality, Rational, Renderer}; +pub use keys::{ExpressionFlags, Interpolation, KeyEase, KeyframeKind, SpatialKey}; +pub use masks::{Mask, MaskAtom, MaskMode, PathVertex, ShapePath}; +pub use matte::{MatteMode, TrackMatte}; +pub use read::{Limits, parse, parse_with_limits}; +pub use settings::{ProjectSettings, TimeDisplay, WorkingSpace}; +pub use shapes::{GradientType, PolystarType, ShapeDetail, ShapeGroup, ShapeItem, Stroke, StrokeCap, StrokeJoin}; +pub use textdoc::{ + AutoKern, Caps, CueType, FontBaseline, Justification, KernRun, Marker, ParagraphRun, StyleRun, TextBox, TextDocument, TextValue, parse_text_dict, +}; +pub use typed::{Comp, Item, ItemKind, KeyLayout, KeyList, Keyframe, Layer, ListHeader, Project, PropEntry, PropGroup, PropNode, Property, TimeBase}; +pub use values::{Orientation, OrientationKey, TimeRemap, TimeRemapKey, Tracked, Transform, ValueSource}; +pub use write::{write, write_to}; diff --git a/crates/aep/src/container/aepx.rs b/crates/aep/src/container/aepx.rs new file mode 100644 index 000000000..a45cbae1d --- /dev/null +++ b/crates/aep/src/container/aepx.rs @@ -0,0 +1,640 @@ +//! Reading the XML form of a project (`.aepx`) into the same chunk tree as an `.aep`. +//! +//! An `.aepx` is the `RIFX` chunk tree written as XML: one element per chunk, named by the chunk +//! id (or the list type of a `LIST`), with the body as hex in a `bdata` attribute. A few chunks +//! are written as text instead (`Utf8` as ``, the numbers `numS` and `ppSn`, the footage +//! alias `alas` as ``), and the XMP packet that follows the container of an `.aep` +//! is the element ``. The mapping, with the cases it was found on, is in +//! `docs/aep-format.md` (section 1.1). +//! +//! The reader is a small XML tokenizer for exactly that subset: elements, attributes, text, the +//! five predefined entities and character references, CDATA, comments and processing +//! instructions. A document type declaration is refused (no entity expansion). It runs without +//! recursion; the nesting depth and the element count are bounded by [`Limits`], and everything +//! it allocates is bounded by the size of the input. + +use std::borrow::Cow; + +use crate::container::chunk::{AepFile, Body, Chunk, LIST, NESTED_IDS}; +use crate::container::error::AepError; +use crate::container::read::{Limits, parse}; + +/// The root element of an `.aepx`. +const ROOT: &[u8] = b"AfterEffectsProject"; +/// The element that holds the XMP packet (the trailer of an `.aep`). +const XMP: &[u8] = b"ProjectXMPMetadata"; +/// The element of a `Utf8` chunk. +const STRING: &[u8] = b"string"; +/// The element of an `alas` chunk (footage alias), written as attributes instead of JSON. +const FILE_REFERENCE: &[u8] = b"fileReference"; + +/// List types whose body is not a chunk sequence: the `.aepx` writes them like data chunks (with +/// `bdata`), the `.aep` as an opaque `LIST`. +const OPAQUE_LIST_TYPES: [[u8; 4]; 1] = [*b"btdk"]; + +/// True when `data` looks like an `.aepx`: optional UTF-8 byte order mark and white space, then +/// an XML declaration or the `AfterEffectsProject` root element. +pub fn is_aepx(data: &[u8]) -> bool { + let rest = data.strip_prefix(&[0xef, 0xbb, 0xbf][..]).unwrap_or(data); + let start = rest.iter().position(|b| !b.is_ascii_whitespace()).unwrap_or(rest.len()); + let rest = rest.get(start..).unwrap_or(&[]); + rest.starts_with(b" Result, AepError> { + if is_aepx(data) { read_aepx(data) } else { parse(data) } +} + +/// Reads an `.aepx` with the default [`Limits`] into the chunk tree an `.aep` of the same project +/// gives. +/// +/// # Errors +/// [`AepError::Xml`] for input that is not well-formed XML of the `.aepx` subset (bad UTF-8, an +/// unknown entity, an unclosed or mismatched tag, invalid or odd-length hex, a number that does +/// not parse, a document type declaration, a root element other than `AfterEffectsProject`); +/// [`AepError::TooDeep`] / [`AepError::TooManyChunks`] beyond the limits. +pub fn read_aepx(data: &[u8]) -> Result, AepError> { + read_aepx_with_limits(data, Limits::default()) +} + +/// Like [`read_aepx`] with explicit [`Limits`]: `max_depth` bounds the element nesting (the root +/// element's children are depth 1), `max_chunks` the number of elements. +/// +/// # Errors +/// See [`read_aepx`]. +pub fn read_aepx_with_limits(data: &[u8], limits: Limits) -> Result, AepError> { + let text = match std::str::from_utf8(data) { + Ok(t) => t.as_bytes(), + Err(e) => return Err(xml(e.valid_up_to(), "the file is not valid UTF-8")), + }; + Reader { data: text, pos: 0, limits, count: 0, stack: Vec::new(), top: None, trailer: None }.run() +} + +fn xml(offset: usize, what: &'static str) -> AepError { + AepError::Xml { offset, what } +} + +/// What an open element becomes when it closes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Kind { + /// The root element: its children are the top-level chunks. + Root, + /// A chunk with this id (or a `LIST` with this list type). + Chunk([u8; 4]), + /// A `Utf8` chunk. + Utf8, + /// An `alas` chunk built from the attributes. + Alias, + /// The XMP packet (only directly below the root). + Xmp, + /// An element without a chunk counterpart (a name that is no chunk id); skipped with its content. + Skip, +} + +struct Frame { + kind: Kind, + /// Offset of the start tag (for errors). + at: usize, + /// The name as written, to match the end tag. + name: (usize, usize), + /// The decoded `bdata` attribute. + bdata: Option>, + /// The attributes of a `fileReference`, in document order. + attrs: Vec<(String, String)>, + /// Collected text (only for `string` and the XMP packet). + text: Vec, + children: Vec>, +} + +struct Reader<'d> { + data: &'d [u8], + pos: usize, + limits: Limits, + count: usize, + stack: Vec, + /// The top-level chunks once the root element has closed. + top: Option>>, + trailer: Option>, +} + +impl Reader<'_> { + fn run(mut self) -> Result, AepError> { + if self.data.starts_with(&[0xef, 0xbb, 0xbf]) { + self.pos = 3; + } + while self.pos < self.data.len() { + if self.rest().starts_with(b"<") { + self.markup()?; + } else { + self.text()?; + } + } + if let Some(open) = self.stack.last() { + return Err(xml(open.at, "an element is not closed before the end of the file")); + } + let chunks = self.top.ok_or_else(|| xml(self.data.len(), "no AfterEffectsProject root element"))?; + Ok(AepFile { chunks, trailer: Cow::Owned(self.trailer.unwrap_or_default()) }) + } + + fn rest(&self) -> &[u8] { + self.data.get(self.pos..).unwrap_or(&[]) + } + + /// The offset of `needle` at or after `from`. + fn find(&self, from: usize, needle: &[u8]) -> Option { + let hay = self.data.get(from..)?; + if needle.is_empty() || hay.len() < needle.len() { + return None; + } + let first = *needle.first()?; + let mut i = 0; + while let Some(p) = hay.get(i..)?.iter().position(|b| *b == first) { + let at = i.checked_add(p)?; + if hay.get(at..)?.starts_with(needle) { + return from.checked_add(at); + } + i = at.checked_add(1)?; + } + None + } + + /// Skips to just after `close`, which must follow. + fn skip_past(&mut self, close: &[u8], what: &'static str) -> Result { + let at = self.find(self.pos, close).ok_or(xml(self.pos, what))?; + self.pos = at.saturating_add(close.len()); + Ok(at) + } + + fn markup(&mut self) -> Result<(), AepError> { + let rest = self.rest(); + if rest.starts_with(b"", "a processing instruction is not closed")?; + } else if rest.starts_with(b"", "a comment is not closed")?; + } else if rest.starts_with(b"", "a CDATA section is not closed")?; + let content = self.data.get(start..end).unwrap_or(&[]); + let at = start; + match self.stack.last_mut() { + Some(frame) => { + if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) { + frame.text.extend_from_slice(content); + } + } + None => return Err(xml(at, "CDATA outside the root element")), + } + } else if rest.starts_with(b" bool { + !(b.is_ascii_whitespace() || matches!(b, b'<' | b'>' | b'/' | b'=' | b'"' | b'\'' | b'&')) + } + + fn name(&mut self) -> Result<(usize, usize), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| !Self::is_name_byte(*b)).unwrap_or(self.rest().len()); + if len == 0 { + return Err(xml(start, "a tag or attribute has no name")); + } + self.pos = start.saturating_add(len); + Ok((start, self.pos)) + } + + fn skip_ws(&mut self) { + let n = self.rest().iter().position(|b| !b.is_ascii_whitespace()).unwrap_or(self.rest().len()); + self.pos = self.pos.saturating_add(n); + } + + fn slice(&self, (a, b): (usize, usize)) -> &[u8] { + self.data.get(a..b).unwrap_or(&[]) + } + + fn start_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(1); + let name = self.name()?; + let mut attrs: Vec<((usize, usize), Vec)> = Vec::new(); + let self_closing = loop { + self.skip_ws(); + let rest = self.rest(); + if rest.starts_with(b"/>") { + self.pos = self.pos.saturating_add(2); + break true; + } + if rest.starts_with(b">") { + self.pos = self.pos.saturating_add(1); + break false; + } + if rest.is_empty() { + return Err(xml(at, "a start tag is not closed")); + } + let key = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b"=") { + return Err(xml(self.pos, "an attribute has no value")); + } + self.pos = self.pos.saturating_add(1); + self.skip_ws(); + let quote = *self.rest().first().ok_or(xml(self.pos, "a start tag is not closed"))?; + if quote != b'"' && quote != b'\'' { + return Err(xml(self.pos, "an attribute value is not quoted")); + } + let vstart = self.pos.saturating_add(1); + let vend = self.find(vstart, &[quote]).ok_or(xml(vstart, "an attribute value is not closed"))?; + let raw = self.data.get(vstart..vend).unwrap_or(&[]); + if raw.contains(&b'<') { + return Err(xml(vstart, "`<` inside an attribute value")); + } + let mut value = Vec::new(); + decode_entities(raw, vstart, &mut value)?; + self.pos = vend.saturating_add(1); + attrs.push((key, value)); + }; + let kind = self.kind_of(name, at)?; + if kind != Kind::Root { + self.count = self.count.saturating_add(1); + if self.count > self.limits.max_chunks { + return Err(AepError::TooManyChunks); + } + // The root's children are depth 1, like the top-level chunks of an `.aep`. + if self.stack.len() > self.limits.max_depth { + return Err(AepError::TooDeep); + } + } + let mut frame = Frame { kind, at, name, bdata: None, attrs: Vec::new(), text: Vec::new(), children: Vec::new() }; + if matches!(kind, Kind::Chunk(_)) { + if let Some((key, value)) = attrs.iter().find(|(k, _)| self.slice(*k) == b"bdata") { + frame.bdata = Some(decode_hex(value, key.1)?); + } + } else if kind == Kind::Alias { + frame.attrs = attrs.iter().map(|(k, v)| (String::from_utf8_lossy(self.slice(*k)).into_owned(), String::from_utf8_lossy(v).into_owned())).collect(); + } + self.stack.push(frame); + if self_closing { + self.close()?; + } + Ok(()) + } + + /// Classifies an element by its name and where it is. + fn kind_of(&self, name: (usize, usize), at: usize) -> Result { + let n = self.slice(name); + let parent = match self.stack.last() { + None => { + if self.top.is_some() { + return Err(xml(at, "a second root element")); + } + if n != ROOT { + return Err(xml(at, "the root element is not AfterEffectsProject")); + } + return Ok(Kind::Root); + } + Some(p) => p.kind, + }; + Ok(match parent { + Kind::Skip | Kind::Utf8 | Kind::Xmp | Kind::Alias => Kind::Skip, + Kind::Root | Kind::Chunk(_) => { + if n == STRING { + Kind::Utf8 + } else if n == FILE_REFERENCE { + Kind::Alias + } else if n == XMP && parent == Kind::Root { + Kind::Xmp + } else if let Some(id) = chunk_id(n) { + Kind::Chunk(id) + } else { + Kind::Skip + } + } + }) + } + + fn end_tag(&mut self) -> Result<(), AepError> { + let at = self.pos; + self.pos = self.pos.saturating_add(2); + let name = self.name()?; + self.skip_ws(); + if !self.rest().starts_with(b">") { + return Err(xml(at, "an end tag is not closed")); + } + self.pos = self.pos.saturating_add(1); + let open = self.stack.last().ok_or(xml(at, "an end tag without a start tag"))?; + if self.slice(open.name) != self.slice(name) { + return Err(xml(at, "an end tag does not match its start tag")); + } + self.close() + } + + /// Closes the innermost element and hands its chunk to the parent. + fn close(&mut self) -> Result<(), AepError> { + let frame = self.stack.pop().ok_or(xml(self.pos, "an end tag without a start tag"))?; + let chunk = match frame.kind { + Kind::Root => { + self.top = Some(frame.children); + return Ok(()); + } + Kind::Skip => return Ok(()), + Kind::Xmp => { + if self.trailer.is_none() { + self.trailer = Some(frame.text); + } + return Ok(()); + } + Kind::Utf8 => Chunk::data(*b"Utf8", frame.text), + Kind::Alias => Chunk::data(*b"alas", alias_json(&frame.attrs).into_bytes()), + Kind::Chunk(id) => chunk_of(id, frame)?, + }; + if let Some(parent) = self.stack.last_mut() { + parent.children.push(chunk); + } + Ok(()) + } + + fn text(&mut self) -> Result<(), AepError> { + let start = self.pos; + let len = self.rest().iter().position(|b| *b == b'<').unwrap_or(self.rest().len()); + let end = start.saturating_add(len); + self.pos = end; + let raw = self.data.get(start..end).unwrap_or(&[]); + match self.stack.last_mut() { + Some(frame) if matches!(frame.kind, Kind::Utf8 | Kind::Xmp) => decode_entities(raw, start, &mut frame.text), + Some(_) => Ok(()), + None if raw.iter().all(u8::is_ascii_whitespace) => Ok(()), + None => Err(xml(start, "text outside the root element")), + } + } +} + +/// The chunk id of an element name: 1 to 4 printable ASCII bytes, padded with spaces (`Pin` is +/// the list type `Pin `). +fn chunk_id(name: &[u8]) -> Option<[u8; 4]> { + if name.is_empty() || name.len() > 4 || !name.iter().all(|b| b.is_ascii_graphic()) { + return None; + } + let mut id = [b' '; 4]; + for (d, s) in id.iter_mut().zip(name) { + *d = *s; + } + Some(id) +} + +/// The chunk of a closed element named by a chunk id. +fn chunk_of(id: [u8; 4], frame: Frame) -> Result, AepError> { + if let Some(bytes) = frame.bdata { + if OPAQUE_LIST_TYPES.contains(&id) { + let pad = if bytes.len() % 2 == 1 { Some(0) } else { None }; + return Ok(Chunk { id: LIST, body: Body::OpaqueList { list_type: id, data: Cow::Owned(bytes) }, pad }); + } + return Ok(Chunk::data(id, bytes)); + } + // Numbers written as text: `0` and `150.000000000`. + if &id == b"numS" || &id == b"ppSn" { + let text = match frame.children.as_slice() { + [only] if only.is(b"Utf8") => only.bytes().unwrap_or(&[]), + _ => return Err(xml(frame.at, "a numS or ppSn element without its number")), + }; + let text = std::str::from_utf8(text).map_err(|_| xml(frame.at, "a number is not text"))?.trim(); + let body = if &id == b"numS" { + // Only `0` is sampled, so whether the field is signed is open: accept both readings of 4 bytes. + let n = text.parse::().ok().filter(|n| (i64::from(i32::MIN)..=i64::from(u32::MAX)).contains(n)); + let n = n.ok_or(xml(frame.at, "numS is not a 32-bit integer"))?; + (n as u32).to_be_bytes().to_vec() + } else { + text.parse::().map_err(|_| xml(frame.at, "ppSn is not a number"))?.to_be_bytes().to_vec() + }; + return Ok(Chunk::data(id, body)); + } + if NESTED_IDS.contains(&id) { + if frame.children.is_empty() { + return Ok(Chunk::data(id, Vec::new())); + } + return Ok(Chunk { id, body: Body::Chunks(frame.children), pad: None }); + } + Ok(Chunk::list(id, frame.children)) +} + +/// Appends `raw` with the predefined entities and character references decoded. +fn decode_entities(raw: &[u8], offset: usize, out: &mut Vec) -> Result<(), AepError> { + let mut i = 0; + while let Some(rest) = raw.get(i..) { + let Some(amp) = rest.iter().position(|b| *b == b'&') else { + out.extend_from_slice(rest); + break; + }; + out.extend_from_slice(rest.get(..amp).unwrap_or(&[])); + let at = offset.saturating_add(i).saturating_add(amp); + let after = rest.get(amp.saturating_add(1)..).unwrap_or(&[]); + let semi = after.iter().take(12).position(|b| *b == b';').ok_or(xml(at, "an entity is not terminated"))?; + let name = after.get(..semi).unwrap_or(&[]); + let ch = match name { + b"amp" => '&', + b"lt" => '<', + b"gt" => '>', + b"quot" => '"', + b"apos" => '\'', + _ => { + let code = if let Some(hex) = name.strip_prefix(b"#x").or_else(|| name.strip_prefix(b"#X")) { + std::str::from_utf8(hex).ok().and_then(|h| u32::from_str_radix(h, 16).ok()) + } else if let Some(dec) = name.strip_prefix(b"#") { + std::str::from_utf8(dec).ok().and_then(|d| d.parse::().ok()) + } else { + return Err(xml(at, "an unknown entity")); + }; + code.filter(|c| *c != 0).and_then(char::from_u32).ok_or(xml(at, "an invalid character reference"))? + } + }; + let mut buf = [0u8; 4]; + out.extend_from_slice(ch.encode_utf8(&mut buf).as_bytes()); + i = i.saturating_add(amp).saturating_add(semi).saturating_add(2); + } + Ok(()) +} + +/// Decodes a `bdata` value (hex digits, two per byte). +fn decode_hex(text: &[u8], offset: usize) -> Result, AepError> { + if text.len() % 2 == 1 { + return Err(xml(offset, "bdata has an odd number of hex digits")); + } + let digit = |b: u8| match b { + b'0'..=b'9' => Some(b - b'0'), + b'a'..=b'f' => Some(b - b'a' + 10), + b'A'..=b'F' => Some(b - b'A' + 10), + _ => None, + }; + let (pairs, _) = text.as_chunks::<2>(); + pairs + .iter() + .map(|[hi, lo]| digit(*hi).zip(digit(*lo)).map(|(h, l)| (h << 4) | l)) + .collect::>>() + .ok_or(xml(offset, "bdata holds a character that is not a hex digit")) +} + +/// The JSON body of an `alas` chunk from the attributes of a `fileReference`, keys in document +/// order (alphabetical in every sample, as in the `.aep`): the two `ascendcount_*` values as +/// numbers, `platform` `Win` as `1`, `target_is_folder` `0` / `1` as `false` / `true`, everything +/// else as a string. +fn alias_json(attrs: &[(String, String)]) -> String { + let mut out = String::from("{"); + for (i, (key, value)) in attrs.iter().enumerate() { + if i > 0 { + out.push(','); + } + json_str(&mut out, key); + out.push(':'); + let number = value.parse::().ok(); + match (key.as_str(), value.as_str()) { + ("ascendcount_base" | "ascendcount_target", _) if number.is_some() => out.push_str(value), + ("platform", "Win") => out.push('1'), + ("platform", _) if number.is_some() => out.push_str(value), + ("target_is_folder", "0" | "false") => out.push_str("false"), + ("target_is_folder", "1" | "true") => out.push_str("true"), + _ => json_str(&mut out, value), + } + } + out.push('}'); + out +} + +fn json_str(out: &mut String, s: &str) { + out.push('"'); + for c in s.chars() { + match c { + '"' => out.push_str("\\\""), + '\\' => out.push_str("\\\\"), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + c if u32::from(c) < 0x20 => out.push_str(&format!("\\u{:04x}", u32::from(c))), + c => out.push(c), + } + } + out.push('"'); +} + +#[cfg(test)] +mod tests { + use super::*; + + fn doc(body: &str) -> String { + format!("\n{body}\n") + } + + #[test] + fn maps_elements_to_chunks() { + let x = doc(concat!( + "\n", + "A & BA-3", + "150.000000000x", + "]]>" + )); + let f = read_aepx(x.as_bytes()).unwrap(); + assert_eq!(f.trailer.as_ref(), b""); + let ids: Vec<_> = f.chunks.iter().map(|c| (c.id, c.list_type())).collect(); + assert_eq!( + ids, + vec![ + (*b"head", None), + (LIST, Some(*b"Pin ")), + (LIST, Some(*b"Pefl")), + (*b"tdsn", None), + (*b"numS", None), + (*b"ppSn", None), + (LIST, Some(*b"btdk")) + ] + ); + assert_eq!(f.chunks[1].children()[0].bytes(), Some(&[1u8][..])); + assert_eq!(f.chunks[1].children()[0].pad, Some(0)); + assert_eq!(f.chunks[3].children()[0].bytes(), Some(&b"A & BA"[..])); + assert_eq!(f.chunks[4].bytes(), Some(&(-3i32).to_be_bytes()[..])); + assert_eq!(f.chunks[5].bytes(), Some(&150f64.to_be_bytes()[..])); + assert_eq!(f.chunks[6].bytes(), Some(&b"ABC"[..])); + // The tree writes as a valid `.aep` that parses back to itself. + let bin = crate::container::write(&f).unwrap(); + assert_eq!(parse(&bin).unwrap(), f); + } + + #[test] + fn file_reference_becomes_alias_json() { + let x = doc(concat!( + "" + )); + let f = read_aepx(x.as_bytes()).unwrap(); + let alas = f.chunks[0].children()[0].bytes().unwrap(); + assert_eq!( + std::str::from_utf8(alas).unwrap(), + r#"{"ascendcount_base":1,"ascendcount_target":1,"fullpath":"C:\\a\\\"b.png","platform":1,"server_name":"N","server_volume_name":"V","target_is_folder":false}"# + ); + } + + #[test] + fn nums_takes_signed_and_unsigned_32_bit_values() { + for (text, bytes) in [("-2147483648", [0x80, 0, 0, 0]), ("4294967295", [0xff; 4]), (" 7 ", [0, 0, 0, 7])] { + let f = read_aepx(doc(&format!("{text}")).as_bytes()).unwrap(); + assert_eq!(f.chunks[0].bytes(), Some(&bytes[..]), "{text}"); + } + for text in ["-2147483649", "4294967296", "1.5"] { + assert!(read_aepx(doc(&format!("{text}")).as_bytes()).is_err(), "{text}"); + } + } + + #[test] + fn detection() { + assert!(is_aepx(b"\xef\xbb\xbf ")); + assert!(is_aepx(b"")); + assert!(!is_aepx(b"RIFX\0\0\0\x04Egg!")); + assert!(parse_any(b"RIFX\0\0\0\x04Egg!").is_ok()); + assert!(parse_any(doc("").as_bytes()).is_ok()); + } + + #[test] + fn bad_input_is_an_error() { + let bad = [ + doc(""), + doc(""), + doc(" "), + doc("�"), + doc("&"), + doc(""), + doc(""), + doc("x"), + doc(""), + doc(""), + doc(""), + "]>".into(), + "".into(), + "".into(), + "".into(), + "junk".into(), + "