From 6ce21609a2073d6ce46d6d17d7a434c0271fcc11 Mon Sep 17 00:00:00 2001 From: Elvis33LE Date: Fri, 9 Oct 2026 08:17:18 +0200 Subject: [PATCH 1/5] raw corpus: real camera files from raw.pixls.us, oracle-tested Seven public-domain files (raw.pixls.us), one per decode path: Canon PowerShot SX50 HS as CR2 and as its DNG conversion, Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2 (10480x7794, crop 10368x7776), Olympus E-1 with a GRBG CFA, and the Olympus E-5 as the documented known-unsupported packed-ORF case (still previews). xtask grows --pixls (single-file downloads, the server redirects /data/ to /download/), a sha256-verified manifest and a --update-manifest path; photocraft-raw gains the corpus feature and oracle tests: format, make/model, sensor size, CFA, black/white levels and the crop are exact, developing yields the crop in 16-bit RGB, deterministic, with sane range coverage. Two real findings already documented by the tests: the PowerShot CR2 and the RX0 ARW carry no as-shot white balance we read, so the grey-world estimate brightens those scenes (the same camera's DNG develops blacks to ~0). Reading the Canon and Sony maker-note WB tags is the follow-up. --- ATTRIBUTION.md | 1 + Cargo.lock | 70 ++++++++++++++- crates/raw/Cargo.toml | 2 +- crates/raw/examples/inspect.rs | 37 ++++++++ crates/raw/tests/raw_corpus.rs | 153 +++++++++++++++++++++++++++++++++ xtask/Cargo.toml | 2 + xtask/pixls.sha256 | 7 ++ xtask/src/corpus.rs | 74 +++++++++++++++- xtask/src/corpus_pins.rs | 18 +++- 9 files changed, 356 insertions(+), 8 deletions(-) create mode 100644 crates/raw/examples/inspect.rs create mode 100644 crates/raw/tests/raw_corpus.rs create mode 100644 xtask/pixls.sha256 diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index fdaf7627d0..303d5b821b 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -85,6 +85,7 @@ next to the files: | `corpus/heif/` (9 files, 0.1 MB) | HEIC/HEIF test files: `heic-rs/` checkerboards, RGB strips and a grid-tiled photo with EXIF and XMP (synthetic pixels encoded by macOS `sips`, each `.ref.png` Apple's decode); `pillow-heif/` the 10-bit RGBA `RGBA_10__29x100.heif` and its source `RGBA_16__29x100.png` | Thomas Braun (heic-rs); Pillow-Heif contributors | [heic-rs `tests/fixtures`](https://github.com/tbraun96/heic-rs/tree/main/tests/fixtures), [pillow-heif `tests/images`](https://github.com/bigcat88/pillow_heif/tree/master/tests/images) | MIT OR Apache-2.0 (heic-rs); BSD-3-Clause (pillow-heif) | | `corpus/exr/` (5 files, 2.3 MB) | The deep OpenEXR test images `11`, `42`, `64`, `multivariate` and `objectid.deep.exr` | Contributors to the OpenEXR Project | [openexr `src/test/bin/test_images`](https://github.com/AcademySoftwareFoundation/openexr/tree/main/src/test/bin/test_images) | BSD-3-Clause | | `corpus/affinity/` (21 files, 2.4 MB) | Public Affinity documents, unmodified: `vector-art/` (9), `afdesignload/` (9), `jac21/` (2) and `asset-store-template/` (1), each with its upstream licence next to it | samuel-etver; Nick Beeuwsaert; Jeremy Cantu; DAIKI (eviltwo) | [vector-art `simple/`](https://github.com/samuel-etver/vector-art/tree/255f8add3c8f0740196e22bd59502b811b532f0b/simple), [AFDesignLoad `testDesigns/`](https://github.com/NickBeeuwsaert/AFDesignLoad/tree/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns), [Jac21/Branding `Logos/JC/DesignerFiles/Affinity`](https://github.com/Jac21/Branding/tree/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity), [AssetStoreTemplate `AssetStoreTemplate/`](https://github.com/eviltwo/AssetStoreTemplate/tree/f671981ee89d15526285b3cc87b4393d3defe644/AssetStoreTemplate) | CC0-1.0 (vector-art); MIT, Copyright (c) 2015 Nick Beeuwsaert; MIT, Copyright (c) 2018 Jeremy Cantu; MIT, Copyright (c) 2024 DAIKI | +| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case | The photographers of the pixls.us raw samples database (per-file credits there) | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain (per the database's release) | | `corpus/pngsuite/` | PngSuite | Willem van Schaik | | Public domain | Files copied into `corpus/` by hand (tiff, exr, raw) must be MIT, BSD or CC0. diff --git a/Cargo.lock b/Cargo.lock index 9dd34b93c9..9fd6dee06b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -630,7 +630,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures", + "cpufeatures 0.3.1", ] [[package]] @@ -639,6 +639,15 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block2" version = "0.5.1" @@ -860,7 +869,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures", + "cpufeatures 0.3.1", "rand_core 0.10.1", ] @@ -1087,6 +1096,15 @@ dependencies = [ "libm", ] +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + [[package]] name = "cpufeatures" version = "0.3.1" @@ -1179,6 +1197,16 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "cursor-icon" version = "1.2.0" @@ -1238,6 +1266,16 @@ dependencies = [ "rayon", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + [[package]] name = "dispatch" version = "0.2.0" @@ -2030,6 +2068,16 @@ dependencies = [ "libc", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "gethostname" version = "1.1.0" @@ -5345,6 +5393,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "shlex" version = "1.3.0" @@ -5972,6 +6031,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "uds_windows" version = "1.2.1" @@ -7221,6 +7286,7 @@ version = "0.5.0" dependencies = [ "serde", "serde_json", + "sha2", "toml", ] diff --git a/crates/raw/Cargo.toml b/crates/raw/Cargo.toml index 5871c96938..55960b33c9 100644 --- a/crates/raw/Cargo.toml +++ b/crates/raw/Cargo.toml @@ -11,7 +11,7 @@ default = [] # Synthetic raw-file generators (DNG / CR2 / TIFF-EP writers and a lossless-JPEG # encoder) for tests and fuzz seeds. testgen = [] -# Real-file corpus tests (opt-in; `cargo xtask test-corpus`), defined with the corpus PR. +# Real-file corpus tests (opt-in; `cargo xtask test-corpus`). Fail when the corpus is missing. corpus = [] [dependencies] diff --git a/crates/raw/examples/inspect.rs b/crates/raw/examples/inspect.rs new file mode 100644 index 0000000000..ca7b136a90 --- /dev/null +++ b/crates/raw/examples/inspect.rs @@ -0,0 +1,37 @@ +//! Prints the raw metadata of files: `cargo run -p photocraft-raw --example inspect -- a.cr2 b.nef` +//! (handy while adding corpus files; the oracle expectations in `tests/raw_corpus.rs` come from it). +use std::env; + +fn main() { + for path in env::args().skip(1) { + let bytes = match std::fs::read(&path) { + Ok(b) => b, + Err(e) => { + println!("{path}: read error {e}"); + continue; + } + }; + match photocraft_raw::decode(&bytes, &photocraft_raw::Limits::default()) { + Ok(s) => { + println!( + "{path}: {:?} make={:?} model={:?} {}x{} samples={} cfa={:?} wb={:?} black={:?} white={:?} crop={:?}..{:?} orientation={} warnings={}", + s.format, + s.make, + s.model, + s.width, + s.height, + s.samples, + s.cfa, + s.camera_wb, + s.black, + s.white, + s.active, + s.crop, + s.orientation, + s.warnings.len(), + ); + } + Err(e) => println!("{path}: decode error {e}"), + } + } +} diff --git a/crates/raw/tests/raw_corpus.rs b/crates/raw/tests/raw_corpus.rs new file mode 100644 index 0000000000..354dec9842 --- /dev/null +++ b/crates/raw/tests/raw_corpus.rs @@ -0,0 +1,153 @@ +//! Real-camera corpus tests against `corpus/pixls` (raw.pixls.us, public domain): one file per +//! decode path, oracle-checked against values measured when the corpus was pinned (the +//! `inspect` example prints them). Opt-in like every real-file corpus: +//! `cargo xtask corpus --pixls` fetches, `cargo xtask test-corpus` runs. +#![cfg(feature = "corpus")] + +use photocraft_raw::{Limits, RawError, RawFormat}; +use std::path::PathBuf; + +fn pixls(name: &str) -> PathBuf { + let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../corpus/pixls").join(name); + if !p.is_file() { + panic!("{} is missing: run `cargo xtask corpus --pixls` (or `--all`) first", p.display()); + } + p +} + +struct Want<'a> { + file: &'static str, + format: RawFormat, + make: &'static str, + model: &'static str, + /// Sensor width and height. + size: (usize, usize), + /// The four CFA colours, row-major. + cfa: [u8; 4], + black: &'a [f32], + white: [f32; 3], +} + +const WANTS: &[Want] = &[ + Want { + file: "IMG_4059.CR2", + format: RawFormat::Cr2, + make: "Canon", + model: "Canon PowerShot SX50 HS", + size: (4176, 3062), + cfa: [0, 1, 1, 2], + black: &[128.0; 4], + white: [4095.0; 3], + }, + // The same camera after DNG Converter: the sensor data round-trips identically. + Want { + file: "CRW_4061.DNG", + format: RawFormat::Dng, + make: "Canon", + model: "Canon PowerShot SX50 HS", + size: (4176, 3062), + cfa: [0, 1, 1, 2], + black: &[127.0], + white: [4095.0; 3], + }, + Want { + file: "JD1_8203.NEF", + format: RawFormat::Nef, + make: "NIKON CORPORATION", + model: "NIKON D3", + size: (4288, 2844), + cfa: [0, 1, 1, 2], + black: &[0.0], + white: [4095.0; 3], + }, + // Sony cRAW: the saturation level comes from the compressed code's range. + Want { + file: "DSC00009.ARW", + format: RawFormat::Arw, + make: "SONY", + model: "DSC-RX0", + size: (4832, 3224), + cfa: [0, 1, 1, 2], + black: &[0.0], + white: [16301.0; 3], + }, + // The G9's high-resolution mode: a 10480 × 7794 sensor cropped to 10368 × 7776. + Want { + file: "P1000475.RW2", + format: RawFormat::Rw2, + make: "Panasonic", + model: "DC-G9", + size: (10480, 7794), + cfa: [0, 1, 1, 2], + black: &[127.0, 128.0, 128.0, 127.0], + white: [4095.0; 3], + }, + // Four Thirds Olympus: GRBG instead of the usual RGGB. + Want { + file: "E_1__C106743_gredos.ORF", + format: RawFormat::Orf, + make: "OLYMPUS CORPORATION", + model: "E-1", + size: (2624, 1966), + cfa: [1, 0, 2, 1], + black: &[65.0; 4], + white: [4095.0; 3], + }, +]; + +#[test] +fn decode_matches_the_measured_oracles() { + for w in WANTS { + let bytes = std::fs::read(pixls(w.file)).unwrap_or_else(|e| panic!("{}: {e}", w.file)); + let s = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file)); + assert_eq!(s.format, w.format, "{}", w.file); + assert_eq!(s.make.as_deref(), Some(w.make), "{}", w.file); + assert_eq!(s.model.as_deref(), Some(w.model), "{}", w.file); + assert_eq!((s.width, s.height), w.size, "{}", w.file); + let cfa = s.cfa.as_ref().unwrap_or_else(|| panic!("{}: no CFA", w.file)); + assert_eq!(cfa.colors, w.cfa, "{}", w.file); + assert_eq!(s.black.values, w.black, "{}", w.file); + for ch in 0..3 { + assert!((s.white[ch] - w.white[ch]).abs() < 0.5, "{} white[{ch}] = {}", w.file, s.white[ch]); + } + } +} + +#[test] +fn developing_produces_the_crop_in_16bit_rgb() { + for w in WANTS { + let bytes = std::fs::read(pixls(w.file)).unwrap_or_else(|e| panic!("{}: {e}", w.file)); + let s = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file)); + let d = photocraft_raw::develop_sensor(&s, &photocraft_raw::DevelopOptions::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file)); + assert_eq!((d.width as usize, d.height as usize), (s.crop.width, s.crop.height), "{}", w.file); + assert_eq!(d.rgb.len(), d.width as usize * d.height as usize * 3, "{}", w.file); + // A real exposure covers a sane share of the range and never leaves it. + let (mut lo, mut hi) = (u16::MAX, 0u16); + for v in &d.rgb { + lo = (*v).min(lo); + hi = (*v).max(hi); + } + // A real exposure covers a sane share of the range and never leaves it. The PowerShot + // CR2 and the RX0 ARW are the documented exceptions: neither carries an as-shot white + // balance we read, so the grey-world estimate brightens these scenes (the PowerShot's + // own DNG, with its file WB, develops blacks to ~0 — reading the maker-note WB of + // Canon and Sony is the follow-up). + let estimated_wb = matches!(w.file, "IMG_4059.CR2" | "DSC00009.ARW"); + let black_cap: u16 = if estimated_wb { 16384 } else { 4096 }; + assert!(lo < black_cap, "{}: developed blacks at {lo}", w.file); + assert!(hi > u16::MAX / 4, "{}: developed highlights at {hi}", w.file); + // Deterministic: a second develop of the same sensor gives the same pixels. + let d2 = photocraft_raw::develop_sensor(&s, &photocraft_raw::DevelopOptions::default()).unwrap(); + assert_eq!(d.rgb, d2.rgb, "{}", w.file); + } +} + +/// The E-5 file is the documented known-unsupported case: later Olympus bodies store packed +/// (compressed) sensor data. The error must say so, never panic, and the file still previews. +#[test] +fn packed_orf_reports_the_documented_gap() { + let bytes = std::fs::read(pixls("_7061961_copy.ORF")).unwrap(); + let err = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_err(); + assert!(matches!(err, RawError::Unsupported(ref m) if m.contains("Olympus compressed")), "{err}"); + assert!(photocraft_raw::embedded_preview(&bytes).is_some(), "the camera's JPEG preview is still reachable"); +} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 25305d5435..42c86af343 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -10,6 +10,8 @@ workspace = true [dependencies] serde_json = { workspace = true } +# corpus manifests (MIT OR Apache-2.0). +sha2 = "0.10" serde = { workspace = true } # perf/budgets.toml and scorecard/*.toml (MIT OR Apache-2.0). toml = "1" diff --git a/xtask/pixls.sha256 b/xtask/pixls.sha256 new file mode 100644 index 0000000000..3878b45356 --- /dev/null +++ b/xtask/pixls.sha256 @@ -0,0 +1,7 @@ +042286653fbae5b085bef4a4e626145385ea6e82e817f166b4904abcef64457c E_1__C106743_gredos.ORF +32e463ae38ae8d8cd1a356af0fc479f2513366c41801d46d4b167cf899656892 CRW_4061.DNG +6c2ddd232f5432fb7bd0e28efc12d66995cedcd847f801fd854bfdfb180db50c IMG_4059.CR2 +7c24fefb1f73ad45c42a0be0078e15b7e91deab1d623efceebfc3c4d0d06135c P1000475.RW2 +880c60c5f611adf6b70de3d099f8433492de3a3d96866d570122f17f0a651fc8 JD1_8203.NEF +ab0d54e3819c9372df0fa98c8686dba0ae87c9d43526a5dd6fd26bd2a664483d _7061961_copy.ORF +f602f8ddeeb9b062f60bfdd3ba0d560c8e54f1f64dbc20e18df5222e2f37a428 DSC00009.ARW diff --git a/xtask/src/corpus.rs b/xtask/src/corpus.rs index 67fa842615..cca751559f 100644 --- a/xtask/src/corpus.rs +++ b/xtask/src/corpus.rs @@ -9,15 +9,24 @@ use crate::pinned::USER_AGENT; use crate::{cargo, root, run}; /// Crates with corpus tests (behind their `corpus` feature). -pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine"]; +pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine", "photocraft-raw"]; /// Corpus crates with a `heif` feature: test-corpus enables it so the HEIF corpus tests run. const HEIF_CRATES: &[&str] = &["photocraft-codecs", "photocraft-io"]; /// Paths whose changes make `test-corpus --changed` run (the file-format and rendering crates). -const CRITICAL: &[&str] = - &["crates/psd/", "crates/io/", "crates/codecs/", "crates/heif/", "crates/compose/", "crates/gpu/", "crates/text/", "crates/format/", "crates/affinity/"]; - +const CRITICAL: &[&str] = &[ + "crates/psd/", + "crates/io/", + "crates/codecs/", + "crates/heif/", + "crates/compose/", + "crates/gpu/", + "crates/text/", + "crates/format/", + "crates/affinity/", + "crates/raw/", +]; /// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --affinity | --photoshop [--local]] [--update-manifest]` pub fn cmd(args: &[&str]) -> Result<(), String> { let update = args.contains(&"--update-manifest"); @@ -41,6 +50,10 @@ pub fn cmd(args: &[&str]) -> Result<(), String> { corpus_pins::HEIF.fetch(update)?; did = true; } + if args.contains(&"--pixls") { + fetch_pixls(update)?; + did = true; + } if args.contains(&"--exr") { corpus_pins::EXR.fetch(update)?; did = true; @@ -78,6 +91,7 @@ fn local_clone() -> Result { /// `corpus/photoshop` is copied from the authoring clone of photocraft-corpus instead. pub fn fetch_all(local: bool) -> Result<(), String> { fetch_pngsuite()?; + fetch_pixls(false)?; for c in corpus_pins::ALL { if local && std::ptr::eq(*c, &corpus_pins::PHOTOSHOP) { c.fetch_local(&local_clone()?, false)?; @@ -222,3 +236,55 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> { } run(c, &format!("cargo test --release --features corpus,heif ({})", crates.join(", "))) } + +/// Fetches `corpus/pixls` (raw.pixls.us single files) and verifies it against +/// `xtask/pixls.sha256`. With `update`, re-hashes the files instead. +pub fn fetch_pixls(update: bool) -> Result<(), String> { + let dest = root().join("corpus").join("pixls"); + std::fs::create_dir_all(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?; + let manifest = root().join("xtask").join("pixls.sha256"); + if update { + let mut out = String::new(); + let mut names: Vec<&str> = corpus_pins::PIXLS_FILES.iter().map(|(_, n)| *n).collect(); + names.sort_unstable(); + for name in names { + let p = dest.join(name); + let sum = file_sha256(&p).map_err(|e| format!("{}: {e}", p.display()))?; + out.push_str(&format!("{sum} {name}\n")); + } + std::fs::write(&manifest, out).map_err(|e| format!("write {}: {e}", manifest.display()))?; + println!("pixls: re-hashed {} files into {}", corpus_pins::PIXLS_FILES.len(), manifest.display()); + return Ok(()); + } + let want: std::collections::HashMap = std::fs::read_to_string(&manifest) + .map_err(|e| format!("read {}: {e} (run `cargo xtask corpus --all` once)", manifest.display()))? + .lines() + .filter_map(|l| l.split_once(" ").map(|(s, n)| (n.to_string(), s.to_string()))) + .collect(); + for (path, name) in corpus_pins::PIXLS_FILES { + let file = dest.join(name); + let have = file_sha256(&file).ok(); + if have.as_deref() == want.get(*name).map(String::as_str) { + continue; + } + let url = format!("{}{}", corpus_pins::PIXLS_BASE, path); + let mut curl = Command::new("curl"); + curl.args(["-fsSL", "--retry", "3", "-A", USER_AGENT, "-o"]).arg(&file).arg(&url); + run(curl, &format!("curl {url}"))?; + let got = file_sha256(&file).map_err(|e| format!("{}: {e}", file.display()))?; + if Some(got.as_str()) != want.get(*name).map(String::as_str) { + return Err(format!("pixls {name}: sha256 mismatch (pin the file you meant, then --update-manifest)")); + } + } + println!("pixls: {} files verified in {}", corpus_pins::PIXLS_FILES.len(), dest.display()); + Ok(()) +} + +/// Hex sha256 of a file (Windows-friendly: no `sha256sum` dependency). +fn file_sha256(path: &std::path::Path) -> Result { + use sha2::{Digest, Sha256}; + let bytes = std::fs::read(path).map_err(|e| e.to_string())?; + let mut h = Sha256::new(); + h.update(&bytes); + Ok(h.finalize().into_iter().map(|b| format!("{b:02x}")).collect()) +} diff --git a/xtask/src/corpus_pins.rs b/xtask/src/corpus_pins.rs index 8e12c69197..ef7bebe047 100644 --- a/xtask/src/corpus_pins.rs +++ b/xtask/src/corpus_pins.rs @@ -224,6 +224,22 @@ pub const AFFINITY: PinnedCorpus = PinnedCorpus { ) }, }; - /// Every pinned corpus, in fetch order. pub const ALL: &[&PinnedCorpus] = &[&PSD_MIXED, &PSD_TOOLS, &PHOTOSHOP, &HEIF, &EXR, &AFFINITY]; + +/// `corpus/pixls`: real camera raws from (released into the public +/// domain / CC0 by their photographers; see ATTRIBUTION.md). One file per decode path +/// `photocraft-raw` supports, plus the packed-ORF known-unsupported oracle. Fetched as single +/// files (the server 301-redirects `/data/` to `/download/`), verified against +/// `xtask/pixls.sha256`. +pub const PIXLS_BASE: &str = "https://raw.pixls.us/data/"; +/// (url path, file name in `corpus/pixls`) +pub const PIXLS_FILES: &[(&str, &str)] = &[ + ("Canon/PowerShot%20SX50%20HS/IMG_4059.CR2", "IMG_4059.CR2"), + ("Canon/PowerShot%20SX50%20HS/CRW_4061.DNG", "CRW_4061.DNG"), + ("Nikon/D3/JD1_8203.NEF", "JD1_8203.NEF"), + ("Sony/DSC-RX0/DSC00009.ARW", "DSC00009.ARW"), + ("Panasonic/DC-G9/P1000475.RW2", "P1000475.RW2"), + ("Olympus/E-1/E_1__C106743_gredos.ORF", "E_1__C106743_gredos.ORF"), + ("Olympus/E-5/_7061961_copy.ORF", "_7061961_copy.ORF"), +]; From 5d0800df7a279bc5305be2f525899c571bc8c9e5 Mon Sep 17 00:00:00 2001 From: Elvis33LE Date: Fri, 9 Oct 2026 10:55:54 +0200 Subject: [PATCH 2/5] pixls corpus: in-house sha256 and pinned.rs, opt-in like the affinity corpus The fetch lives in pinned.rs now, reusing its manifest discipline and the xtask sha256 module (the sha2 dependency is gone). --pixls is opt-in: not part of --all or the CI corpus job (CORPUS_CRATES no longer includes photocraft-raw; test-corpus --pixls fetches and adds it), matching how the Affinity corpus is handled. The usage line names the flag, and the attribution row links each sample's folder in the database so its licence can be checked per file. --- ATTRIBUTION.md | 2 +- Cargo.lock | 70 ++----------------------------------------- xtask/Cargo.toml | 2 -- xtask/src/corpus.rs | 73 +++++++++------------------------------------ xtask/src/pinned.rs | 49 ++++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 130 deletions(-) diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 303d5b821b..4b1a1ff324 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -85,7 +85,7 @@ next to the files: | `corpus/heif/` (9 files, 0.1 MB) | HEIC/HEIF test files: `heic-rs/` checkerboards, RGB strips and a grid-tiled photo with EXIF and XMP (synthetic pixels encoded by macOS `sips`, each `.ref.png` Apple's decode); `pillow-heif/` the 10-bit RGBA `RGBA_10__29x100.heif` and its source `RGBA_16__29x100.png` | Thomas Braun (heic-rs); Pillow-Heif contributors | [heic-rs `tests/fixtures`](https://github.com/tbraun96/heic-rs/tree/main/tests/fixtures), [pillow-heif `tests/images`](https://github.com/bigcat88/pillow_heif/tree/master/tests/images) | MIT OR Apache-2.0 (heic-rs); BSD-3-Clause (pillow-heif) | | `corpus/exr/` (5 files, 2.3 MB) | The deep OpenEXR test images `11`, `42`, `64`, `multivariate` and `objectid.deep.exr` | Contributors to the OpenEXR Project | [openexr `src/test/bin/test_images`](https://github.com/AcademySoftwareFoundation/openexr/tree/main/src/test/bin/test_images) | BSD-3-Clause | | `corpus/affinity/` (21 files, 2.4 MB) | Public Affinity documents, unmodified: `vector-art/` (9), `afdesignload/` (9), `jac21/` (2) and `asset-store-template/` (1), each with its upstream licence next to it | samuel-etver; Nick Beeuwsaert; Jeremy Cantu; DAIKI (eviltwo) | [vector-art `simple/`](https://github.com/samuel-etver/vector-art/tree/255f8add3c8f0740196e22bd59502b811b532f0b/simple), [AFDesignLoad `testDesigns/`](https://github.com/NickBeeuwsaert/AFDesignLoad/tree/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns), [Jac21/Branding `Logos/JC/DesignerFiles/Affinity`](https://github.com/Jac21/Branding/tree/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity), [AssetStoreTemplate `AssetStoreTemplate/`](https://github.com/eviltwo/AssetStoreTemplate/tree/f671981ee89d15526285b3cc87b4393d3defe644/AssetStoreTemplate) | CC0-1.0 (vector-art); MIT, Copyright (c) 2015 Nick Beeuwsaert; MIT, Copyright (c) 2018 Jeremy Cantu; MIT, Copyright (c) 2024 DAIKI | -| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case | The photographers of the pixls.us raw samples database (per-file credits there) | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain (per the database's release) | +| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case. Each file (and its licence, per the database) is reachable at its path under [raw.pixls.us/data](https://raw.pixls.us/data/): [Canon SX50 HS](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/), [Nikon D3](https://raw.pixls.us/data/Nikon/D3/), [Sony DSC-RX0](https://raw.pixls.us/data/Sony/DSC-RX0/), [Panasonic DC-G9](https://raw.pixls.us/data/Panasonic/DC-G9/), [Olympus E-1](https://raw.pixls.us/data/Olympus/E-1/), [Olympus E-5](https://raw.pixls.us/data/Olympus/E-5/) | The photographers of the pixls.us raw samples database (per-file credits there) | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain, per file as shown in the [database listing](https://raw.pixls.us) | | `corpus/pngsuite/` | PngSuite | Willem van Schaik | | Public domain | Files copied into `corpus/` by hand (tiff, exr, raw) must be MIT, BSD or CC0. diff --git a/Cargo.lock b/Cargo.lock index 9fd6dee06b..9dd34b93c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -630,7 +630,7 @@ dependencies = [ "cc", "cfg-if", "constant_time_eq", - "cpufeatures 0.3.1", + "cpufeatures", ] [[package]] @@ -639,15 +639,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" -[[package]] -name = "block-buffer" -version = "0.10.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" -dependencies = [ - "generic-array", -] - [[package]] name = "block2" version = "0.5.1" @@ -869,7 +860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", - "cpufeatures 0.3.1", + "cpufeatures", "rand_core 0.10.1", ] @@ -1096,15 +1087,6 @@ dependencies = [ "libm", ] -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - [[package]] name = "cpufeatures" version = "0.3.1" @@ -1197,16 +1179,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" -[[package]] -name = "crypto-common" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" -dependencies = [ - "generic-array", - "typenum", -] - [[package]] name = "cursor-icon" version = "1.2.0" @@ -1266,16 +1238,6 @@ dependencies = [ "rayon", ] -[[package]] -name = "digest" -version = "0.10.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" -dependencies = [ - "block-buffer", - "crypto-common", -] - [[package]] name = "dispatch" version = "0.2.0" @@ -2068,16 +2030,6 @@ dependencies = [ "libc", ] -[[package]] -name = "generic-array" -version = "0.14.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" -dependencies = [ - "typenum", - "version_check", -] - [[package]] name = "gethostname" version = "1.1.0" @@ -5393,17 +5345,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures 0.2.17", - "digest", -] - [[package]] name = "shlex" version = "1.3.0" @@ -6031,12 +5972,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" -[[package]] -name = "typenum" -version = "1.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" - [[package]] name = "uds_windows" version = "1.2.1" @@ -7286,7 +7221,6 @@ version = "0.5.0" dependencies = [ "serde", "serde_json", - "sha2", "toml", ] diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index 42c86af343..25305d5435 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -10,8 +10,6 @@ workspace = true [dependencies] serde_json = { workspace = true } -# corpus manifests (MIT OR Apache-2.0). -sha2 = "0.10" serde = { workspace = true } # perf/budgets.toml and scorecard/*.toml (MIT OR Apache-2.0). toml = "1" diff --git a/xtask/src/corpus.rs b/xtask/src/corpus.rs index cca751559f..424452e0e8 100644 --- a/xtask/src/corpus.rs +++ b/xtask/src/corpus.rs @@ -9,7 +9,9 @@ use crate::pinned::USER_AGENT; use crate::{cargo, root, run}; /// Crates with corpus tests (behind their `corpus` feature). -pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine", "photocraft-raw"]; +pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine"]; +/// Opt-in corpus crates: fetched and tested only with `test-corpus --pixls`. +pub const PIXLS_CRATES: &[&str] = &["photocraft-raw"]; /// Corpus crates with a `heif` feature: test-corpus enables it so the HEIF corpus tests run. const HEIF_CRATES: &[&str] = &["photocraft-codecs", "photocraft-io"]; @@ -27,7 +29,7 @@ const CRITICAL: &[&str] = &[ "crates/affinity/", "crates/raw/", ]; -/// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --affinity | --photoshop [--local]] [--update-manifest]` +/// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --pixls | --affinity | --photoshop [--local]] [--update-manifest]` pub fn cmd(args: &[&str]) -> Result<(), String> { let update = args.contains(&"--update-manifest"); let mut did = false; @@ -51,7 +53,7 @@ pub fn cmd(args: &[&str]) -> Result<(), String> { did = true; } if args.contains(&"--pixls") { - fetch_pixls(update)?; + crate::pinned::fetch_pixls(update)?; did = true; } if args.contains(&"--exr") { @@ -91,7 +93,6 @@ fn local_clone() -> Result { /// `corpus/photoshop` is copied from the authoring clone of photocraft-corpus instead. pub fn fetch_all(local: bool) -> Result<(), String> { fetch_pngsuite()?; - fetch_pixls(false)?; for c in corpus_pins::ALL { if local && std::ptr::eq(*c, &corpus_pins::PHOTOSHOP) { c.fetch_local(&local_clone()?, false)?; @@ -210,18 +211,24 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> { "-p" | "--package" => { let name = it.next().ok_or("-p needs a crate name")?; let full = if name.starts_with("photocraft-") { (*name).to_string() } else { format!("photocraft-{name}") }; - if !CORPUS_CRATES.contains(&full.as_str()) { - return Err(format!("{full} has no corpus tests (crates: {})", CORPUS_CRATES.join(", "))); + if !CORPUS_CRATES.contains(&full.as_str()) && !PIXLS_CRATES.contains(&full.as_str()) { + return Err(format!("{full} has no corpus tests (crates: {} or the opt-in {})", CORPUS_CRATES.join(", "), PIXLS_CRATES.join(", "))); } crates.push(full); } - "--changed" | "--local" => {} + "--changed" | "--local" | "--pixls" => {} other => return Err(format!("test-corpus: unknown argument `{other}`")), } } if crates.is_empty() { crates = CORPUS_CRATES.iter().map(|s| (*s).to_string()).collect(); } + // `--pixls` opts in (and fetches): the raw corpus tests are not part of CI until a + // maintainer decides they belong there. + if ours.contains(&"--pixls") || crates.iter().any(|c| PIXLS_CRATES.contains(&c.as_str())) { + crates.push("photocraft-raw".to_string()); + crate::pinned::fetch_pixls(false)?; + } fetch_all(ours.contains(&"--local"))?; let mut c = cargo(); c.args(["test", "--release", "--lib", "--tests"]); @@ -236,55 +243,3 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> { } run(c, &format!("cargo test --release --features corpus,heif ({})", crates.join(", "))) } - -/// Fetches `corpus/pixls` (raw.pixls.us single files) and verifies it against -/// `xtask/pixls.sha256`. With `update`, re-hashes the files instead. -pub fn fetch_pixls(update: bool) -> Result<(), String> { - let dest = root().join("corpus").join("pixls"); - std::fs::create_dir_all(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?; - let manifest = root().join("xtask").join("pixls.sha256"); - if update { - let mut out = String::new(); - let mut names: Vec<&str> = corpus_pins::PIXLS_FILES.iter().map(|(_, n)| *n).collect(); - names.sort_unstable(); - for name in names { - let p = dest.join(name); - let sum = file_sha256(&p).map_err(|e| format!("{}: {e}", p.display()))?; - out.push_str(&format!("{sum} {name}\n")); - } - std::fs::write(&manifest, out).map_err(|e| format!("write {}: {e}", manifest.display()))?; - println!("pixls: re-hashed {} files into {}", corpus_pins::PIXLS_FILES.len(), manifest.display()); - return Ok(()); - } - let want: std::collections::HashMap = std::fs::read_to_string(&manifest) - .map_err(|e| format!("read {}: {e} (run `cargo xtask corpus --all` once)", manifest.display()))? - .lines() - .filter_map(|l| l.split_once(" ").map(|(s, n)| (n.to_string(), s.to_string()))) - .collect(); - for (path, name) in corpus_pins::PIXLS_FILES { - let file = dest.join(name); - let have = file_sha256(&file).ok(); - if have.as_deref() == want.get(*name).map(String::as_str) { - continue; - } - let url = format!("{}{}", corpus_pins::PIXLS_BASE, path); - let mut curl = Command::new("curl"); - curl.args(["-fsSL", "--retry", "3", "-A", USER_AGENT, "-o"]).arg(&file).arg(&url); - run(curl, &format!("curl {url}"))?; - let got = file_sha256(&file).map_err(|e| format!("{}: {e}", file.display()))?; - if Some(got.as_str()) != want.get(*name).map(String::as_str) { - return Err(format!("pixls {name}: sha256 mismatch (pin the file you meant, then --update-manifest)")); - } - } - println!("pixls: {} files verified in {}", corpus_pins::PIXLS_FILES.len(), dest.display()); - Ok(()) -} - -/// Hex sha256 of a file (Windows-friendly: no `sha256sum` dependency). -fn file_sha256(path: &std::path::Path) -> Result { - use sha2::{Digest, Sha256}; - let bytes = std::fs::read(path).map_err(|e| e.to_string())?; - let mut h = Sha256::new(); - h.update(&bytes); - Ok(h.finalize().into_iter().map(|b| format!("{b:02x}")).collect()) -} diff --git a/xtask/src/pinned.rs b/xtask/src/pinned.rs index 130e19d345..917c0b864b 100644 --- a/xtask/src/pinned.rs +++ b/xtask/src/pinned.rs @@ -263,3 +263,52 @@ impl PinnedCorpus { self.install(&found, |f| Some(src.join(&f[up.prefix.len()..])), &extras, true, &format!("copied from the local clone {}", clone.display())) } } + +/// Hex sha256 of a corpus file (reused by every corpus fetch, `xtask/pixls.sha256` included). +fn file_hex(path: &std::path::Path) -> Option { + std::fs::read(path).ok().map(|b| crate::sha256::hex(&b)) +} + +/// Fetches `corpus/pixls` — the raw.pixls.us camera files pinned in `corpus_pins::PIXLS_FILES` — +/// and verifies them against `xtask/pixls.sha256`, the same manifest form every corpus uses. +/// raw.pixls.us serves single files and redirects `/data/` to `/download/`, so this is a plain +/// per-file download rather than a git upstream; the manifest discipline is the same +/// (`--update-manifest` re-pins). Opt-in: not part of `--all` or CI until a maintainer says so. +pub fn fetch_pixls(update: bool) -> Result<(), String> { + let dest = crate::root().join("corpus").join("pixls"); + std::fs::create_dir_all(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?; + let manifest = crate::root().join("xtask").join("pixls.sha256"); + if update { + let mut names: Vec<&str> = crate::corpus_pins::PIXLS_FILES.iter().map(|(_, n)| *n).collect(); + names.sort_unstable(); + let mut out = String::new(); + for name in names { + let sum = file_hex(&dest.join(name)).ok_or_else(|| format!("pixls {name}: not downloaded yet"))?; + out.push_str(&format!("{sum} {name}\n")); + } + std::fs::write(&manifest, out).map_err(|e| format!("write {}: {e}", manifest.display()))?; + println!("pixls: re-hashed {} files into {}", crate::corpus_pins::PIXLS_FILES.len(), manifest.display()); + return Ok(()); + } + let want: std::collections::HashMap = std::fs::read_to_string(&manifest) + .map_err(|e| format!("read {}: {e} (run `cargo xtask corpus --pixls` once)", manifest.display()))? + .lines() + .filter_map(|l| l.split_once(" ").map(|(s, n)| (n.to_string(), s.to_string()))) + .collect(); + for (path, name) in crate::corpus_pins::PIXLS_FILES { + let file = dest.join(name); + if file_hex(&file).as_deref() == want.get(*name).map(String::as_str) { + continue; + } + let url = format!("{}{}", crate::corpus_pins::PIXLS_BASE, path); + let mut curl = Command::new("curl"); + curl.args(["-fsSL", "--retry", "3", "-A", USER_AGENT, "-o"]).arg(&file).arg(&url); + run(curl, &format!("curl {url}"))?; + let got = file_hex(&file).ok_or_else(|| format!("pixls {name}: unreadable"))?; + if Some(got.as_str()) != want.get(*name).map(String::as_str) { + return Err(format!("pixls {name}: sha256 mismatch (pin the file you meant, then --update-manifest)")); + } + } + println!("pixls: {} files verified in {}", crate::corpus_pins::PIXLS_FILES.len(), dest.display()); + Ok(()) +} From 3bfaef668c47b77297c89b1f6110ac4ffbbd5c95 Mon Sep 17 00:00:00 2001 From: Elvis33LE Date: Fri, 9 Oct 2026 12:04:56 +0200 Subject: [PATCH 3/5] pixls: usage and status text, per-file licence links, dedup, --changed for raw The xtask usage line and the corpus status listing name --pixls and corpus/pixls; the attribution row links each of the seven files' database entries so the licence can be checked per file; -p raw no longer pushes the crate into the list twice; and test-corpus --changed runs (and fetches) the raw corpus when a file under crates/raw/ changed, matching its CRITICAL entry. --- ATTRIBUTION.md | 2 +- xtask/src/corpus.rs | 14 ++++++++++---- xtask/src/main.rs | 4 ++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 4b1a1ff324..03eb256e95 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -86,6 +86,6 @@ next to the files: | `corpus/exr/` (5 files, 2.3 MB) | The deep OpenEXR test images `11`, `42`, `64`, `multivariate` and `objectid.deep.exr` | Contributors to the OpenEXR Project | [openexr `src/test/bin/test_images`](https://github.com/AcademySoftwareFoundation/openexr/tree/main/src/test/bin/test_images) | BSD-3-Clause | | `corpus/affinity/` (21 files, 2.4 MB) | Public Affinity documents, unmodified: `vector-art/` (9), `afdesignload/` (9), `jac21/` (2) and `asset-store-template/` (1), each with its upstream licence next to it | samuel-etver; Nick Beeuwsaert; Jeremy Cantu; DAIKI (eviltwo) | [vector-art `simple/`](https://github.com/samuel-etver/vector-art/tree/255f8add3c8f0740196e22bd59502b811b532f0b/simple), [AFDesignLoad `testDesigns/`](https://github.com/NickBeeuwsaert/AFDesignLoad/tree/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns), [Jac21/Branding `Logos/JC/DesignerFiles/Affinity`](https://github.com/Jac21/Branding/tree/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity), [AssetStoreTemplate `AssetStoreTemplate/`](https://github.com/eviltwo/AssetStoreTemplate/tree/f671981ee89d15526285b3cc87b4393d3defe644/AssetStoreTemplate) | CC0-1.0 (vector-art); MIT, Copyright (c) 2015 Nick Beeuwsaert; MIT, Copyright (c) 2018 Jeremy Cantu; MIT, Copyright (c) 2024 DAIKI | | `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case. Each file (and its licence, per the database) is reachable at its path under [raw.pixls.us/data](https://raw.pixls.us/data/): [Canon SX50 HS](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/), [Nikon D3](https://raw.pixls.us/data/Nikon/D3/), [Sony DSC-RX0](https://raw.pixls.us/data/Sony/DSC-RX0/), [Panasonic DC-G9](https://raw.pixls.us/data/Panasonic/DC-G9/), [Olympus E-1](https://raw.pixls.us/data/Olympus/E-1/), [Olympus E-5](https://raw.pixls.us/data/Olympus/E-5/) | The photographers of the pixls.us raw samples database (per-file credits there) | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain, per file as shown in the [database listing](https://raw.pixls.us) | -| `corpus/pngsuite/` | PngSuite | Willem van Schaik | | Public domain | +| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case. Per file (each CC0, per its database entry): [IMG_4059.CR2](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/IMG_4059.CR2), [CRW_4061.DNG](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/CRW_4061.DNG), [JD1_8203.NEF](https://raw.pixls.us/data/Nikon/D3/JD1_8203.NEF), [DSC00009.ARW](https://raw.pixls.us/data/Sony/DSC-RX0/DSC00009.ARW), [P1000475.RW2](https://raw.pixls.us/data/Panasonic/DC-G9/P1000475.RW2), [E_1__C106743_gredos.ORF](https://raw.pixls.us/data/Olympus/E-1/E_1__C106743_gredos.ORF), [_7061961_copy.ORF](https://raw.pixls.us/data/Olympus/E-5/_7061961_copy.ORF) | The photographers of the pixls.us raw samples database | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain, per file as linked | Files copied into `corpus/` by hand (tiff, exr, raw) must be MIT, BSD or CC0. diff --git a/xtask/src/corpus.rs b/xtask/src/corpus.rs index 424452e0e8..84754a0391 100644 --- a/xtask/src/corpus.rs +++ b/xtask/src/corpus.rs @@ -134,7 +134,8 @@ fn list() { corpus/affinity/ [{}] 21 public Affinity 1–3 documents (CC0, MIT): vector-art, AFDesignLoad, Jac21/Branding, AssetStoreTemplate; manifest xtask/affinity-corpus.sha256. Fetch: --affinity corpus/pngsuite/ [{}] PngSuite (public domain), {PNGSUITE_URL}. Fetch: --pngsuite - corpus/tiff/, corpus/raw/ optional, copied in by hand + corpus/pixls/ [opt-in] real camera raws from raw.pixls.us (public domain), one per decode + path plus the known-unsupported packed ORF. Fetch: --pixls (opt-in) Pins: xtask/src/corpus_pins.rs. Moving one: change it, then -- --update-manifest.", corpus.display(), @@ -195,6 +196,7 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> { Some(i) => (&args[..i], &args[i + 1..]), None => (args, &[][..]), }; + let mut raw_changed = false; if ours.contains(&"--changed") { let changed = changed_files()?; let hits: Vec<&String> = changed.iter().filter(|f| CRITICAL.iter().any(|c| f.starts_with(c))).collect(); @@ -203,6 +205,7 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> { return Ok(()); } println!("test-corpus --changed: {} critical files changed (e.g. {}); running the corpus tests", hits.len(), hits[0]); + raw_changed = hits.iter().any(|f| f.starts_with("crates/raw/")); } let mut crates: Vec = Vec::new(); let mut it = ours.iter(); @@ -223,10 +226,13 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> { if crates.is_empty() { crates = CORPUS_CRATES.iter().map(|s| (*s).to_string()).collect(); } - // `--pixls` opts in (and fetches): the raw corpus tests are not part of CI until a - // maintainer decides they belong there. - if ours.contains(&"--pixls") || crates.iter().any(|c| PIXLS_CRATES.contains(&c.as_str())) { + // `--pixls` opts in (and fetches), and a raw change under --changed does too: the raw + // corpus tests are not part of the default set until a maintainer decides otherwise. + let wants_pixls = ours.contains(&"--pixls") || crates.iter().any(|c| PIXLS_CRATES.contains(&c.as_str())) || raw_changed; + if wants_pixls && !crates.iter().any(|c| c == "photocraft-raw") { crates.push("photocraft-raw".to_string()); + } + if wants_pixls { crate::pinned::fetch_pixls(false)?; } fetch_all(ours.contains(&"--local"))?; diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 634a6aa565..9399c74614 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -25,9 +25,9 @@ commands: layers enforce the crate dependency layering (plan/architecture.md §3) wasm cargo check --target wasm32-unknown-unknown for the wasm-safe crates ci fmt --check, clippy -D warnings, test, layers, wasm (stops at first failure) - corpus [--all | --pngsuite | --psd | --psd-tools | --photoshop] [--local] [--update-manifest] + corpus [--all | --pngsuite | --psd | --psd-tools | --pixls | --photoshop] [--local] [--update-manifest] show where test corpora live and their pins (xtask/src/corpus_pins.rs), or fetch - them into corpus/ (pinned commits, sha256-verified; --all = every corpus; + them into corpus/ (pinned commits, sha256-verified; --all = every corpus but pixls, --photoshop --local copies from ../photocraft-corpus or $PHOTOCRAFT_CORPUS_REPO) test-corpus [-p ]... [--changed] [--local] [-- ] fetch every corpus, then cargo test --release --features corpus on the corpus From 2e41caf8979aec9648f07c94798216671e831662 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 9 Oct 2026 09:29:58 -0400 Subject: [PATCH 4/5] ATTRIBUTION: keep the PngSuite row and one pixls row Co-Authored-By: Claude Opus 5.5 --- ATTRIBUTION.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 03eb256e95..de62e87a4a 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -85,7 +85,7 @@ next to the files: | `corpus/heif/` (9 files, 0.1 MB) | HEIC/HEIF test files: `heic-rs/` checkerboards, RGB strips and a grid-tiled photo with EXIF and XMP (synthetic pixels encoded by macOS `sips`, each `.ref.png` Apple's decode); `pillow-heif/` the 10-bit RGBA `RGBA_10__29x100.heif` and its source `RGBA_16__29x100.png` | Thomas Braun (heic-rs); Pillow-Heif contributors | [heic-rs `tests/fixtures`](https://github.com/tbraun96/heic-rs/tree/main/tests/fixtures), [pillow-heif `tests/images`](https://github.com/bigcat88/pillow_heif/tree/master/tests/images) | MIT OR Apache-2.0 (heic-rs); BSD-3-Clause (pillow-heif) | | `corpus/exr/` (5 files, 2.3 MB) | The deep OpenEXR test images `11`, `42`, `64`, `multivariate` and `objectid.deep.exr` | Contributors to the OpenEXR Project | [openexr `src/test/bin/test_images`](https://github.com/AcademySoftwareFoundation/openexr/tree/main/src/test/bin/test_images) | BSD-3-Clause | | `corpus/affinity/` (21 files, 2.4 MB) | Public Affinity documents, unmodified: `vector-art/` (9), `afdesignload/` (9), `jac21/` (2) and `asset-store-template/` (1), each with its upstream licence next to it | samuel-etver; Nick Beeuwsaert; Jeremy Cantu; DAIKI (eviltwo) | [vector-art `simple/`](https://github.com/samuel-etver/vector-art/tree/255f8add3c8f0740196e22bd59502b811b532f0b/simple), [AFDesignLoad `testDesigns/`](https://github.com/NickBeeuwsaert/AFDesignLoad/tree/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns), [Jac21/Branding `Logos/JC/DesignerFiles/Affinity`](https://github.com/Jac21/Branding/tree/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity), [AssetStoreTemplate `AssetStoreTemplate/`](https://github.com/eviltwo/AssetStoreTemplate/tree/f671981ee89d15526285b3cc87b4393d3defe644/AssetStoreTemplate) | CC0-1.0 (vector-art); MIT, Copyright (c) 2015 Nick Beeuwsaert; MIT, Copyright (c) 2018 Jeremy Cantu; MIT, Copyright (c) 2024 DAIKI | -| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case. Each file (and its licence, per the database) is reachable at its path under [raw.pixls.us/data](https://raw.pixls.us/data/): [Canon SX50 HS](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/), [Nikon D3](https://raw.pixls.us/data/Nikon/D3/), [Sony DSC-RX0](https://raw.pixls.us/data/Sony/DSC-RX0/), [Panasonic DC-G9](https://raw.pixls.us/data/Panasonic/DC-G9/), [Olympus E-1](https://raw.pixls.us/data/Olympus/E-1/), [Olympus E-5](https://raw.pixls.us/data/Olympus/E-5/) | The photographers of the pixls.us raw samples database (per-file credits there) | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain, per file as shown in the [database listing](https://raw.pixls.us) | +| `corpus/pngsuite/` | PngSuite | Willem van Schaik | | Public domain | | `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case. Per file (each CC0, per its database entry): [IMG_4059.CR2](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/IMG_4059.CR2), [CRW_4061.DNG](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/CRW_4061.DNG), [JD1_8203.NEF](https://raw.pixls.us/data/Nikon/D3/JD1_8203.NEF), [DSC00009.ARW](https://raw.pixls.us/data/Sony/DSC-RX0/DSC00009.ARW), [P1000475.RW2](https://raw.pixls.us/data/Panasonic/DC-G9/P1000475.RW2), [E_1__C106743_gredos.ORF](https://raw.pixls.us/data/Olympus/E-1/E_1__C106743_gredos.ORF), [_7061961_copy.ORF](https://raw.pixls.us/data/Olympus/E-5/_7061961_copy.ORF) | The photographers of the pixls.us raw samples database | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain, per file as linked | Files copied into `corpus/` by hand (tiff, exr, raw) must be MIT, BSD or CC0. From beb80bc5dcc401753aae4d25e5fbff33548055ac Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 9 Oct 2026 13:56:22 +0000 Subject: [PATCH 5/5] Carry #1636: document test-corpus --pixls Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UHgFEHEsefoQoBParH47JA --- xtask/src/main.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 9399c74614..50f3b6b13b 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -29,9 +29,10 @@ commands: show where test corpora live and their pins (xtask/src/corpus_pins.rs), or fetch them into corpus/ (pinned commits, sha256-verified; --all = every corpus but pixls, --photoshop --local copies from ../photocraft-corpus or $PHOTOCRAFT_CORPUS_REPO) - test-corpus [-p ]... [--changed] [--local] [-- ] + test-corpus [-p ]... [--changed] [--pixls] [--local] [-- ] fetch every corpus, then cargo test --release --features corpus on the corpus - crates; --changed runs only if psd/io/codecs/compose/gpu/text/format changed; + crates; --changed runs only if psd/io/codecs/compose/gpu/text/format/raw changed; + --pixls (or -p raw) also fetches corpus/pixls and runs the raw corpus tests; --local takes corpus/photoshop from the photocraft-corpus authoring clone stats [--exact] count tests and lines per crate (--exact: ask the test harness via `-- --list`) parity Photoshop menu parity; rewrites docs/parity.md