From d6fcd4a62b37d089dd9514c10143b5457ac1716c Mon Sep 17 00:00:00 2001 From: Kristian Myram Date: Thu, 15 Oct 2026 20:53:31 +0200 Subject: [PATCH] S2.60: measured tracked-frame cost; bench times memory attention A tracked frame is the encode (2.4 s) plus memory attention (2.4 s with a full bank), about 5 s on 16 threads, not one encode as the docs said. Docs corrected; chunked attention tried and reverted. Co-Authored-By: Claude Sonnet 5.5 --- crates/sam2/examples/sam2_bench.rs | 14 ++++++++++++++ docs/roto-brush-sam2.md | 17 +++++++++++++---- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/crates/sam2/examples/sam2_bench.rs b/crates/sam2/examples/sam2_bench.rs index ea871a4f0..a4e78feb0 100644 --- a/crates/sam2/examples/sam2_bench.rs +++ b/crates/sam2/examples/sam2_bench.rs @@ -56,6 +56,20 @@ fn run() -> Result<(), Box> { let _ = effectcraft_sam2::neck::sine_position(128, h, w)?; } println!("sine positions for the four levels: {:.0} ms", t.elapsed().as_secs_f64() * 1e3); + // What a tracked frame adds to the encode: memory attention over the memory bank (7 memory frames and + // 16 object pointers of 4 tokens), then the memory encoder. + let weights = effectcraft_sam2::weights::Weights::open(&file)?; + let attn = effectcraft_sam2::memattn::MemoryAttention::load(&weights)?; + let cur = Tensor::randn(0f32, 1.0, (4096, 256), &Device::Cpu)?; + let cur_pos = Tensor::randn(0f32, 1.0, (4096, 256), &Device::Cpu)?; + let m = 7 * 4096 + 64; + let mem = Tensor::randn(0f32, 1.0, (m, 64), &Device::Cpu)?; + let mem_pos = Tensor::randn(0f32, 1.0, (m, 64), &Device::Cpu)?; + for run in 0..2 { + let t = Instant::now(); + let _ = attn.forward(&cur, &cur_pos, &mem, &mem_pos, 64)?; + println!("memory attention (run {run}, {m} memory tokens): {:.0} ms", t.elapsed().as_secs_f64() * 1e3); + } println!("threads: {}", std::thread::available_parallelism().map_or(1, |n| n.get())); Ok(()) } diff --git a/docs/roto-brush-sam2.md b/docs/roto-brush-sam2.md index e61f49be7..580553655 100644 --- a/docs/roto-brush-sam2.md +++ b/docs/roto-brush-sam2.md @@ -64,10 +64,19 @@ the CPU. SAM 2 is prompted with at most three foreground and three background points per frame: it was trained on a handful of clicks, and measured confidence collapses with many (0.99 with 1–7 points, 0.25 with 16, 0.00 with 32). -## Speed (this PC: 16 threads, CPU only) -A 1024 × 1024 frame takes about **3.4 s** to encode (release build); a click on an encoded frame about 120 ms. Tracking a -clip costs one encode per frame, so a 300-frame clip takes on the order of 17 minutes on a CPU. Candle has no AMD/Vulkan -backend; GPU work is Phase 6 of the plan (the early Hiera blocks are memory-bound and have headroom on the CPU too). +## Speed (this PC: 16 threads, CPU only, release build) +| Step | Time | +|---|---| +| Encode one 1024 × 1024 frame | about **2.4 s** (3.4 s before the speed work: a parallel GELU, cached position encodings, pooling without permutes) | +| A click on an encoded frame | about 105 ms | +| Memory attention on a tracked frame (full bank of 7 memory frames) | about **2.4 s** | +| A tracked frame in all | about **5 s** (the first six have fewer memories) | + +A 300-frame clip therefore takes on the order of 25 minutes on this CPU. (An earlier version of this page said one encode +per frame; the memory attention is as large as the encode.) What was tried and did not help: chunking the queries of the +global and memory attention to keep scores in cache. Candle has no AMD/Vulkan backend, so a GPU path is future work (a Burn +or hand-written wgpu route); `cargo run --release -p effectcraft-sam2 --example sam2_bench` and `--example op_bench` time the +pieces. ## Benchmark `effectcraft-cli bench --roto [--frames N] [--sam2 model.safetensors] [--mobilesam mobile_sam.pt]` runs synthetic clips