From 2f75b21ac3c0ac66d43e9e0a98dd5bdaaacb7486 Mon Sep 17 00:00:00 2001 From: dev-null-undefined Date: Sat, 10 Oct 2026 13:05:06 +0200 Subject: [PATCH] LR-IMP-FORMATS: ARW6 corpus pins two files (FF Compressed, APS-C HQ) `cargo xtask corpus --download` now fetches one full-frame Compressed and one APS-C Compressed HQ file (about 94 MB) instead of all 12 (about 630 MB); the corpus test checks those two and the lossless-sibling regression test is gone, since nothing would download those files. docs/arw6-compression.md keeps the full table with source paths and SHA-256, says which two xtask fetches and how to fetch the rest with `curl -L`, and records the CC0 1.0 statement of RawDB's set record (accessed 2026-10-10). Co-Authored-By: Claude --- crates/raw/tests/corpus.rs | 23 ++--------------- docs/arw6-compression.md | 34 ++++++++++++------------ xtask/src/main.rs | 53 ++------------------------------------ 3 files changed, 22 insertions(+), 88 deletions(-) diff --git a/crates/raw/tests/corpus.rs b/crates/raw/tests/corpus.rs index cb8928678..3170742a6 100644 --- a/crates/raw/tests/corpus.rs +++ b/crates/raw/tests/corpus.rs @@ -881,37 +881,18 @@ fn verify_sensor_arrays(cases: &[(&str, usize, usize, u64, u64)]) -> usize { seen } -/// Reference sums from the oracle-verified Phase 0 model (docs/arw6-compression.md); the files are CC0 from RawDB. +/// Reference sums from the oracle-verified Phase 0 model (docs/arw6-compression.md); the files are CC0 from RawDB, +/// one per geometry and compression mode. #[test] fn corpus_sony_arw6_samples() { let cases: &[(&str, usize, usize, u64, u64)] = &[ ("arw6-sony-ilce7rm6-apsc-land-hq.arw", 6592, 4372, 85834467900, 1323814215465139776), - ("arw6-sony-ilce7rm6-apsc-land-c.arw", 6592, 4372, 100710478599, 1516298925086164077), - ("arw6-sony-ilce7rm6-ff-land-hq.arw", 10016, 6672, 258417758289, 9667602918876623332), ("arw6-sony-ilce7rm6-ff-land-c.arw", 10016, 6672, 258558260019, 9665041696390559268), - ("arw6-sony-ilce7rm6-apsc-port-hq.arw", 6592, 4372, 126965386051, 1647947621898118992), - ("arw6-sony-ilce7rm6-apsc-port-c.arw", 6592, 4372, 122629012213, 1601616875702755341), - ("arw6-sony-ilce7rm6-ff-port-hq.arw", 10016, 6672, 242772794178, 8043470668590476975), - ("arw6-sony-ilce7rm6-ff-port-c.arw", 10016, 6672, 241868995482, 8014659803009871380), ]; let seen = verify_sensor_arrays(cases); eprintln!("verified {seen} Sony ARW6 sensor arrays"); } -/// The lossless siblings keep decoding through the LJ92 quad-tile path. Regression values from LightCraft's own -/// LJ92 path, 2026-10-10 (no independent reference). -#[test] -fn corpus_sony_ilce7rm6_lossless_samples() { - let cases: &[(&str, usize, usize, u64, u64)] = &[ - ("arw-sony-ilce7rm6-apsc-land-lossless.arw", 6656, 4608, 42685586690, 664300694689609738), - ("arw-sony-ilce7rm6-apsc-port-lossless.arw", 6656, 4608, 62443944469, 818441184988649303), - ("arw-sony-ilce7rm6-ff-land-lossless.arw", 10240, 7168, 128728791716, 4922490451480041632), - ("arw-sony-ilce7rm6-ff-port-lossless.arw", 10240, 7168, 120718314835, 4098302368966213583), - ]; - let seen = verify_sensor_arrays(cases); - eprintln!("verified {seen} Sony ILCE-7RM6 lossless sensor arrays"); -} - /// The container rule for TIFF "shells" (a small IFD0 next to a private raw block) must not touch real /// raws: every corpus file keeps the format its maker implies, and none is described as a private block. #[test] diff --git a/docs/arw6-compression.md b/docs/arw6-compression.md index 9f263051c..793991a26 100644 --- a/docs/arw6-compression.md +++ b/docs/arw6-compression.md @@ -57,22 +57,24 @@ LibRaw assumes the full-frame grid (s = 0) for every tile. On the APS-C crop tha ### Corpus -| File | Source path (RawDB, ILCE-7RM6) | Size (bytes) | SHA-256 | -|---|---|---|---| -| `arw6-sony-ilce7rm6-apsc-land-hq.arw` | `arw6/ILCE-7RM6_APSC_land_RAW-C-HD.ARW` | 29 720 576 | `b60a7e752d2a8102ccbe6eebcfdbf533c6b0f2545739090873e9e8eb32cc0637` | -| `arw6-sony-ilce7rm6-apsc-land-c.arw` | `arw6/ILCE-7RM6_APSC_land_RAW-C.ARW` | 30 900 224 | `1c803304e63ec508b2b5f9d59d4a7e30ed279315b35fa4c5d75f959bed62198b` | -| `arw6-sony-ilce7rm6-apsc-port-hq.arw` | `arw6/ILCE-7RM6_APSC_port_RAW-C-HD.ARW` | 28 758 016 | `0b6d782571e86868670f613de9fa1f1364623b24fbfd21b704b2c516d7a1c1cb` | -| `arw6-sony-ilce7rm6-apsc-port-c.arw` | `arw6/ILCE-7RM6_APSC_port_RAW-C.ARW` | 29 044 736 | `c41674a485ffcf6d873e20e545ae62936192d94e95be3d53548e93bc543ecfd4` | -| `arw6-sony-ilce7rm6-ff-land-hq.arw` | `arw6/ILCE-7RM6_FF_land_RAW-C-HD.ARW` | 63 451 136 | `559fa26501d7813e56afecde2aea3d9965a99228162b194752d986193abb172c` | -| `arw6-sony-ilce7rm6-ff-land-c.arw` | `arw6/ILCE-7RM6_FF_land_RAW-C.ARW` | 63 844 352 | `d900d6cbe838be864269ad54de4a33fcac7eb3a3d4f91009739f770b8f85a23b` | -| `arw6-sony-ilce7rm6-ff-port-hq.arw` | `arw6/ILCE-7RM6_FF_port_RAW-C-HD.ARW` | 62 640 128 | `81c709633cb8f6340e612c3f53bb6ec6b0cccfe5b713c67efb1e977ece9e266d` | -| `arw6-sony-ilce7rm6-ff-port-c.arw` | `arw6/ILCE-7RM6_FF_port_RAW-C.ARW` | 63 168 512 | `f4448485629c3f82ff1955fd046404350e9a1f09b346038574b89c074d7c50fd` | -| `arw-sony-ilce7rm6-apsc-land-lossless.arw` | `raw_modes/ILCE-7RM6_APSC_land_RAW.ARW` | 44 167 168 | `7aaf0b3941509d7364a6f03844783155ebc912d246d1c456e5cdb4693ec09fbc` | -| `arw-sony-ilce7rm6-apsc-port-lossless.arw` | `raw_modes/ILCE-7RM6_APSC_port_RAW.ARW` | 43 692 032 | `0f011933ee96525a39710daa50b59d39702b4ec5934fa0206742e2128c55f366` | -| `arw-sony-ilce7rm6-ff-land-lossless.arw` | `raw_modes/ILCE-7RM6_FF_land_RAW.ARW` | 100 851 712 | `5cb3c870bc961cf6b079d56c860d0cdf11acd1afd04e5fb8dbe041bcfd2d6201` | -| `arw-sony-ilce7rm6-ff-port-lossless.arw` | `raw_modes/ILCE-7RM6_FF_port_RAW.ARW` | 100 884 480 | `b1a4026aa6adbaa27973b8b3426d8b4aa0c1cae1845cf5102359327d3b069691` | +| File | Source path (RawDB, ILCE-7RM6) | Size (bytes) | SHA-256 | Fetched by | +|---|---|---|---|---| +| `arw6-sony-ilce7rm6-apsc-land-hq.arw` | `arw6/ILCE-7RM6_APSC_land_RAW-C-HD.ARW` | 29 720 576 | `b60a7e752d2a8102ccbe6eebcfdbf533c6b0f2545739090873e9e8eb32cc0637` | `cargo xtask corpus --download` | +| `arw6-sony-ilce7rm6-apsc-land-c.arw` | `arw6/ILCE-7RM6_APSC_land_RAW-C.ARW` | 30 900 224 | `1c803304e63ec508b2b5f9d59d4a7e30ed279315b35fa4c5d75f959bed62198b` | `curl -L` | +| `arw6-sony-ilce7rm6-apsc-port-hq.arw` | `arw6/ILCE-7RM6_APSC_port_RAW-C-HD.ARW` | 28 758 016 | `0b6d782571e86868670f613de9fa1f1364623b24fbfd21b704b2c516d7a1c1cb` | `curl -L` | +| `arw6-sony-ilce7rm6-apsc-port-c.arw` | `arw6/ILCE-7RM6_APSC_port_RAW-C.ARW` | 29 044 736 | `c41674a485ffcf6d873e20e545ae62936192d94e95be3d53548e93bc543ecfd4` | `curl -L` | +| `arw6-sony-ilce7rm6-ff-land-hq.arw` | `arw6/ILCE-7RM6_FF_land_RAW-C-HD.ARW` | 63 451 136 | `559fa26501d7813e56afecde2aea3d9965a99228162b194752d986193abb172c` | `curl -L` | +| `arw6-sony-ilce7rm6-ff-land-c.arw` | `arw6/ILCE-7RM6_FF_land_RAW-C.ARW` | 63 844 352 | `d900d6cbe838be864269ad54de4a33fcac7eb3a3d4f91009739f770b8f85a23b` | `cargo xtask corpus --download` | +| `arw6-sony-ilce7rm6-ff-port-hq.arw` | `arw6/ILCE-7RM6_FF_port_RAW-C-HD.ARW` | 62 640 128 | `81c709633cb8f6340e612c3f53bb6ec6b0cccfe5b713c67efb1e977ece9e266d` | `curl -L` | +| `arw6-sony-ilce7rm6-ff-port-c.arw` | `arw6/ILCE-7RM6_FF_port_RAW-C.ARW` | 63 168 512 | `f4448485629c3f82ff1955fd046404350e9a1f09b346038574b89c074d7c50fd` | `curl -L` | +| `arw-sony-ilce7rm6-apsc-land-lossless.arw` | `raw_modes/ILCE-7RM6_APSC_land_RAW.ARW` | 44 167 168 | `7aaf0b3941509d7364a6f03844783155ebc912d246d1c456e5cdb4693ec09fbc` | `curl -L` | +| `arw-sony-ilce7rm6-apsc-port-lossless.arw` | `raw_modes/ILCE-7RM6_APSC_port_RAW.ARW` | 43 692 032 | `0f011933ee96525a39710daa50b59d39702b4ec5934fa0206742e2128c55f366` | `curl -L` | +| `arw-sony-ilce7rm6-ff-land-lossless.arw` | `raw_modes/ILCE-7RM6_FF_land_RAW.ARW` | 100 851 712 | `5cb3c870bc961cf6b079d56c860d0cdf11acd1afd04e5fb8dbe041bcfd2d6201` | `curl -L` | +| `arw-sony-ilce7rm6-ff-port-lossless.arw` | `raw_modes/ILCE-7RM6_FF_port_RAW.ARW` | 100 884 480 | `b1a4026aa6adbaa27973b8b3426d8b4aa0c1cae1845cf5102359327d3b069691` | `curl -L` | -These 12 files are CC0 1.0 per the RawDB record of the set (uploaded by abbradar; ); the server answers 303 to a signed object URL, so fetch with `curl -L` from `https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/`. They live in the gitignored corpus; `cargo xtask corpus --download` fetches them and checks each against its SHA-256, pinned in `RAW_SAMPLES` (`xtask/src/main.rs`). `cargo test -p lightcraft-raw corpus_sony -- --nocapture` verifies the decoder's sensor sum and position-weighted sum of each available file. The 8 ARW6 files are checked against the Phase 0 reference sums; the 4 lossless siblings go through the existing LJ92 path and their sums are regression values from LightCraft's own decoder. The test skips absent files. +These 12 files are CC0 1.0. RawDB's record of the set, (the data behind , accessed 2026-10-10), says `"license":"CC0 1.0"` for the set (uploaded by abbradar, 2026-08-29) and again for each of the 12 files, next to the SHA-256 values in the table. + +They live in the gitignored corpus. `cargo xtask corpus --download` fetches two of them, one per geometry and compression mode (full frame Compressed, APS-C HQ), and checks each against its SHA-256, pinned in `RAW_SAMPLES` (`xtask/src/main.rs`). Fetch the other ten by hand: the server answers 303 to a signed object URL, so `curl -L -o corpus/raw/ https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/`, then compare with the SHA-256 above. `cargo test -p lightcraft-raw corpus_sony -- --nocapture` checks the decoder's sensor sum and position-weighted sum of the two pinned files against the Phase 0 reference sums and skips absent files. Nine further full-frame HQ photos from the maintainer's A7R VI were compared with the oracle locally; they are private and never committed or published. All 9 private FF photos (ISO 100-12800, incl. two near-black night shots) decode bit-exact over the full frame (0 mismatching pixels of 66.8 MP each). @@ -84,7 +86,7 @@ Unconditional synthetic tests (encode -> decode round trips for both geometries, |---|---| | ILCE-7RM6 Compressed and HQ, full frame, landscape and portrait | 4 files, exact against the reference sums | | ILCE-7RM6 Compressed and HQ, APS-C crop, landscape and portrait | 4 files, exact against the reference sums (interior-exact against LibRaw after the 6-row shift; LibRaw's own edge rows differ) | -| ILCE-7RM6 lossless (existing LJ92 path) | 4 files, regression values | +| ILCE-7RM6 lossless (existing LJ92 path) | 4 files decode; not in the corpus test | | Private A7R VI HQ photos | 9 files, bit-exact over the full frame against the oracle (local) | ## Not covered diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 91063e748..14b71d4de 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -775,67 +775,18 @@ const RAW_SAMPLES: &[(&str, &str, &str)] = &[ "https://raw.pixls.us/getfile.php/7807/nice/Fujifilm%20-%20X-T50%20-%2014bit%20compressed%20%283:2%29.RAF", "3a8a9ac6d92274969aaa1f55ea669191b98bb8f588d00ec8ec17afd8b3963dea", ), - // Sony ILCE-7RM6: ARW6 Compressed (RAW-C) / Compressed HQ (RAW-C-HD) and the lossless siblings; CC0 1.0, RawDB. + // Sony ILCE-7RM6 ARW6, CC0 1.0 (RawDB): one full-frame Compressed (RAW-C) and one APS-C Compressed HQ (RAW-C-HD) file; + // docs/arw6-compression.md lists the rest of the set. ( "arw6-sony-ilce7rm6-apsc-land-hq.arw", "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/arw6/ILCE-7RM6_APSC_land_RAW-C-HD.ARW", "b60a7e752d2a8102ccbe6eebcfdbf533c6b0f2545739090873e9e8eb32cc0637", ), - ( - "arw6-sony-ilce7rm6-apsc-land-c.arw", - "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/arw6/ILCE-7RM6_APSC_land_RAW-C.ARW", - "1c803304e63ec508b2b5f9d59d4a7e30ed279315b35fa4c5d75f959bed62198b", - ), - ( - "arw6-sony-ilce7rm6-apsc-port-hq.arw", - "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/arw6/ILCE-7RM6_APSC_port_RAW-C-HD.ARW", - "0b6d782571e86868670f613de9fa1f1364623b24fbfd21b704b2c516d7a1c1cb", - ), - ( - "arw6-sony-ilce7rm6-apsc-port-c.arw", - "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/arw6/ILCE-7RM6_APSC_port_RAW-C.ARW", - "c41674a485ffcf6d873e20e545ae62936192d94e95be3d53548e93bc543ecfd4", - ), - ( - "arw6-sony-ilce7rm6-ff-land-hq.arw", - "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/arw6/ILCE-7RM6_FF_land_RAW-C-HD.ARW", - "559fa26501d7813e56afecde2aea3d9965a99228162b194752d986193abb172c", - ), ( "arw6-sony-ilce7rm6-ff-land-c.arw", "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/arw6/ILCE-7RM6_FF_land_RAW-C.ARW", "d900d6cbe838be864269ad54de4a33fcac7eb3a3d4f91009739f770b8f85a23b", ), - ( - "arw6-sony-ilce7rm6-ff-port-hq.arw", - "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/arw6/ILCE-7RM6_FF_port_RAW-C-HD.ARW", - "81c709633cb8f6340e612c3f53bb6ec6b0cccfe5b713c67efb1e977ece9e266d", - ), - ( - "arw6-sony-ilce7rm6-ff-port-c.arw", - "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/arw6/ILCE-7RM6_FF_port_RAW-C.ARW", - "f4448485629c3f82ff1955fd046404350e9a1f09b346038574b89c074d7c50fd", - ), - ( - "arw-sony-ilce7rm6-apsc-land-lossless.arw", - "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/raw_modes/ILCE-7RM6_APSC_land_RAW.ARW", - "7aaf0b3941509d7364a6f03844783155ebc912d246d1c456e5cdb4693ec09fbc", - ), - ( - "arw-sony-ilce7rm6-apsc-port-lossless.arw", - "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/raw_modes/ILCE-7RM6_APSC_port_RAW.ARW", - "0f011933ee96525a39710daa50b59d39702b4ec5934fa0206742e2128c55f366", - ), - ( - "arw-sony-ilce7rm6-ff-land-lossless.arw", - "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/raw_modes/ILCE-7RM6_FF_land_RAW.ARW", - "5cb3c870bc961cf6b079d56c860d0cdf11acd1afd04e5fb8dbe041bcfd2d6201", - ), - ( - "arw-sony-ilce7rm6-ff-port-lossless.arw", - "https://rawdb.dnglab.org/api/download/Sony/ILCE-7RM6/raw_modes/ILCE-7RM6_FF_port_RAW.ARW", - "b1a4026aa6adbaa27973b8b3426d8b4aa0c1cae1845cf5102359327d3b069691", - ), ( "rw2-panasonic-g9-b.rw2", "https://raw.pixls.us/getfile.php/2348/nice/Panasonic%20-%20DC-G9%20-%204:3.RW2",