diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md
index 2a0eaa75e1..7bbf2a1f91 100644
--- a/ATTRIBUTION.md
+++ b/ATTRIBUTION.md
@@ -87,5 +87,6 @@ next to the files:
| `corpus/exr/` (5 files, 2.3 MB) | The deep OpenEXR test images `11`, `42`, `64`, `multivariate` and `objectid.deep.exr` | Contributors to the OpenEXR Project | [openexr `src/test/bin/test_images`](https://github.com/AcademySoftwareFoundation/openexr/tree/main/src/test/bin/test_images) | BSD-3-Clause |
| `corpus/affinity/` (21 files, 2.4 MB) | Public Affinity documents, unmodified: `vector-art/` (9), `afdesignload/` (9), `jac21/` (2) and `asset-store-template/` (1), each with its upstream licence next to it | samuel-etver; Nick Beeuwsaert; Jeremy Cantu; DAIKI (eviltwo) | [vector-art `simple/`](https://github.com/samuel-etver/vector-art/tree/255f8add3c8f0740196e22bd59502b811b532f0b/simple), [AFDesignLoad `testDesigns/`](https://github.com/NickBeeuwsaert/AFDesignLoad/tree/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns), [Jac21/Branding `Logos/JC/DesignerFiles/Affinity`](https://github.com/Jac21/Branding/tree/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity), [AssetStoreTemplate `AssetStoreTemplate/`](https://github.com/eviltwo/AssetStoreTemplate/tree/f671981ee89d15526285b3cc87b4393d3defe644/AssetStoreTemplate) | CC0-1.0 (vector-art); MIT, Copyright (c) 2015 Nick Beeuwsaert; MIT, Copyright (c) 2018 Jeremy Cantu; MIT, Copyright (c) 2024 DAIKI |
| `corpus/pngsuite/` | PngSuite | Willem van Schaik | | Public domain |
+| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case. Per file (each CC0, per its database entry): [IMG_4059.CR2](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/IMG_4059.CR2), [CRW_4061.DNG](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/CRW_4061.DNG), [JD1_8203.NEF](https://raw.pixls.us/data/Nikon/D3/JD1_8203.NEF), [DSC00009.ARW](https://raw.pixls.us/data/Sony/DSC-RX0/DSC00009.ARW), [P1000475.RW2](https://raw.pixls.us/data/Panasonic/DC-G9/P1000475.RW2), [E_1__C106743_gredos.ORF](https://raw.pixls.us/data/Olympus/E-1/E_1__C106743_gredos.ORF), [_7061961_copy.ORF](https://raw.pixls.us/data/Olympus/E-5/_7061961_copy.ORF) | The photographers of the pixls.us raw samples database | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain, per file as linked |
Files copied into `corpus/` by hand (tiff, exr, raw) must be MIT, BSD or CC0.
diff --git a/crates/raw/Cargo.toml b/crates/raw/Cargo.toml
index 5871c96938..55960b33c9 100644
--- a/crates/raw/Cargo.toml
+++ b/crates/raw/Cargo.toml
@@ -11,7 +11,7 @@ default = []
# Synthetic raw-file generators (DNG / CR2 / TIFF-EP writers and a lossless-JPEG
# encoder) for tests and fuzz seeds.
testgen = []
-# Real-file corpus tests (opt-in; `cargo xtask test-corpus`), defined with the corpus PR.
+# Real-file corpus tests (opt-in; `cargo xtask test-corpus`). Fail when the corpus is missing.
corpus = []
[dependencies]
diff --git a/crates/raw/examples/inspect.rs b/crates/raw/examples/inspect.rs
new file mode 100644
index 0000000000..ca7b136a90
--- /dev/null
+++ b/crates/raw/examples/inspect.rs
@@ -0,0 +1,37 @@
+//! Prints the raw metadata of files: `cargo run -p photocraft-raw --example inspect -- a.cr2 b.nef`
+//! (handy while adding corpus files; the oracle expectations in `tests/raw_corpus.rs` come from it).
+use std::env;
+
+fn main() {
+ for path in env::args().skip(1) {
+ let bytes = match std::fs::read(&path) {
+ Ok(b) => b,
+ Err(e) => {
+ println!("{path}: read error {e}");
+ continue;
+ }
+ };
+ match photocraft_raw::decode(&bytes, &photocraft_raw::Limits::default()) {
+ Ok(s) => {
+ println!(
+ "{path}: {:?} make={:?} model={:?} {}x{} samples={} cfa={:?} wb={:?} black={:?} white={:?} crop={:?}..{:?} orientation={} warnings={}",
+ s.format,
+ s.make,
+ s.model,
+ s.width,
+ s.height,
+ s.samples,
+ s.cfa,
+ s.camera_wb,
+ s.black,
+ s.white,
+ s.active,
+ s.crop,
+ s.orientation,
+ s.warnings.len(),
+ );
+ }
+ Err(e) => println!("{path}: decode error {e}"),
+ }
+ }
+}
diff --git a/crates/raw/tests/raw_corpus.rs b/crates/raw/tests/raw_corpus.rs
new file mode 100644
index 0000000000..354dec9842
--- /dev/null
+++ b/crates/raw/tests/raw_corpus.rs
@@ -0,0 +1,153 @@
+//! Real-camera corpus tests against `corpus/pixls` (raw.pixls.us, public domain): one file per
+//! decode path, oracle-checked against values measured when the corpus was pinned (the
+//! `inspect` example prints them). Opt-in like every real-file corpus:
+//! `cargo xtask corpus --pixls` fetches, `cargo xtask test-corpus` runs.
+#![cfg(feature = "corpus")]
+
+use photocraft_raw::{Limits, RawError, RawFormat};
+use std::path::PathBuf;
+
+fn pixls(name: &str) -> PathBuf {
+ let p = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../corpus/pixls").join(name);
+ if !p.is_file() {
+ panic!("{} is missing: run `cargo xtask corpus --pixls` (or `--all`) first", p.display());
+ }
+ p
+}
+
+struct Want<'a> {
+ file: &'static str,
+ format: RawFormat,
+ make: &'static str,
+ model: &'static str,
+ /// Sensor width and height.
+ size: (usize, usize),
+ /// The four CFA colours, row-major.
+ cfa: [u8; 4],
+ black: &'a [f32],
+ white: [f32; 3],
+}
+
+const WANTS: &[Want] = &[
+ Want {
+ file: "IMG_4059.CR2",
+ format: RawFormat::Cr2,
+ make: "Canon",
+ model: "Canon PowerShot SX50 HS",
+ size: (4176, 3062),
+ cfa: [0, 1, 1, 2],
+ black: &[128.0; 4],
+ white: [4095.0; 3],
+ },
+ // The same camera after DNG Converter: the sensor data round-trips identically.
+ Want {
+ file: "CRW_4061.DNG",
+ format: RawFormat::Dng,
+ make: "Canon",
+ model: "Canon PowerShot SX50 HS",
+ size: (4176, 3062),
+ cfa: [0, 1, 1, 2],
+ black: &[127.0],
+ white: [4095.0; 3],
+ },
+ Want {
+ file: "JD1_8203.NEF",
+ format: RawFormat::Nef,
+ make: "NIKON CORPORATION",
+ model: "NIKON D3",
+ size: (4288, 2844),
+ cfa: [0, 1, 1, 2],
+ black: &[0.0],
+ white: [4095.0; 3],
+ },
+ // Sony cRAW: the saturation level comes from the compressed code's range.
+ Want {
+ file: "DSC00009.ARW",
+ format: RawFormat::Arw,
+ make: "SONY",
+ model: "DSC-RX0",
+ size: (4832, 3224),
+ cfa: [0, 1, 1, 2],
+ black: &[0.0],
+ white: [16301.0; 3],
+ },
+ // The G9's high-resolution mode: a 10480 × 7794 sensor cropped to 10368 × 7776.
+ Want {
+ file: "P1000475.RW2",
+ format: RawFormat::Rw2,
+ make: "Panasonic",
+ model: "DC-G9",
+ size: (10480, 7794),
+ cfa: [0, 1, 1, 2],
+ black: &[127.0, 128.0, 128.0, 127.0],
+ white: [4095.0; 3],
+ },
+ // Four Thirds Olympus: GRBG instead of the usual RGGB.
+ Want {
+ file: "E_1__C106743_gredos.ORF",
+ format: RawFormat::Orf,
+ make: "OLYMPUS CORPORATION",
+ model: "E-1",
+ size: (2624, 1966),
+ cfa: [1, 0, 2, 1],
+ black: &[65.0; 4],
+ white: [4095.0; 3],
+ },
+];
+
+#[test]
+fn decode_matches_the_measured_oracles() {
+ for w in WANTS {
+ let bytes = std::fs::read(pixls(w.file)).unwrap_or_else(|e| panic!("{}: {e}", w.file));
+ let s = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file));
+ assert_eq!(s.format, w.format, "{}", w.file);
+ assert_eq!(s.make.as_deref(), Some(w.make), "{}", w.file);
+ assert_eq!(s.model.as_deref(), Some(w.model), "{}", w.file);
+ assert_eq!((s.width, s.height), w.size, "{}", w.file);
+ let cfa = s.cfa.as_ref().unwrap_or_else(|| panic!("{}: no CFA", w.file));
+ assert_eq!(cfa.colors, w.cfa, "{}", w.file);
+ assert_eq!(s.black.values, w.black, "{}", w.file);
+ for ch in 0..3 {
+ assert!((s.white[ch] - w.white[ch]).abs() < 0.5, "{} white[{ch}] = {}", w.file, s.white[ch]);
+ }
+ }
+}
+
+#[test]
+fn developing_produces_the_crop_in_16bit_rgb() {
+ for w in WANTS {
+ let bytes = std::fs::read(pixls(w.file)).unwrap_or_else(|e| panic!("{}: {e}", w.file));
+ let s = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file));
+ let d = photocraft_raw::develop_sensor(&s, &photocraft_raw::DevelopOptions::default()).unwrap_or_else(|e| panic!("{}: {e}", w.file));
+ assert_eq!((d.width as usize, d.height as usize), (s.crop.width, s.crop.height), "{}", w.file);
+ assert_eq!(d.rgb.len(), d.width as usize * d.height as usize * 3, "{}", w.file);
+ // A real exposure covers a sane share of the range and never leaves it.
+ let (mut lo, mut hi) = (u16::MAX, 0u16);
+ for v in &d.rgb {
+ lo = (*v).min(lo);
+ hi = (*v).max(hi);
+ }
+ // A real exposure covers a sane share of the range and never leaves it. The PowerShot
+ // CR2 and the RX0 ARW are the documented exceptions: neither carries an as-shot white
+ // balance we read, so the grey-world estimate brightens these scenes (the PowerShot's
+ // own DNG, with its file WB, develops blacks to ~0 — reading the maker-note WB of
+ // Canon and Sony is the follow-up).
+ let estimated_wb = matches!(w.file, "IMG_4059.CR2" | "DSC00009.ARW");
+ let black_cap: u16 = if estimated_wb { 16384 } else { 4096 };
+ assert!(lo < black_cap, "{}: developed blacks at {lo}", w.file);
+ assert!(hi > u16::MAX / 4, "{}: developed highlights at {hi}", w.file);
+ // Deterministic: a second develop of the same sensor gives the same pixels.
+ let d2 = photocraft_raw::develop_sensor(&s, &photocraft_raw::DevelopOptions::default()).unwrap();
+ assert_eq!(d.rgb, d2.rgb, "{}", w.file);
+ }
+}
+
+/// The E-5 file is the documented known-unsupported case: later Olympus bodies store packed
+/// (compressed) sensor data. The error must say so, never panic, and the file still previews.
+#[test]
+fn packed_orf_reports_the_documented_gap() {
+ let bytes = std::fs::read(pixls("_7061961_copy.ORF")).unwrap();
+ let err = photocraft_raw::decode(&bytes, &Limits::default()).unwrap_err();
+ assert!(matches!(err, RawError::Unsupported(ref m) if m.contains("Olympus compressed")), "{err}");
+ assert!(photocraft_raw::embedded_preview(&bytes).is_some(), "the camera's JPEG preview is still reachable");
+}
diff --git a/xtask/pixls.sha256 b/xtask/pixls.sha256
new file mode 100644
index 0000000000..3878b45356
--- /dev/null
+++ b/xtask/pixls.sha256
@@ -0,0 +1,7 @@
+042286653fbae5b085bef4a4e626145385ea6e82e817f166b4904abcef64457c E_1__C106743_gredos.ORF
+32e463ae38ae8d8cd1a356af0fc479f2513366c41801d46d4b167cf899656892 CRW_4061.DNG
+6c2ddd232f5432fb7bd0e28efc12d66995cedcd847f801fd854bfdfb180db50c IMG_4059.CR2
+7c24fefb1f73ad45c42a0be0078e15b7e91deab1d623efceebfc3c4d0d06135c P1000475.RW2
+880c60c5f611adf6b70de3d099f8433492de3a3d96866d570122f17f0a651fc8 JD1_8203.NEF
+ab0d54e3819c9372df0fa98c8686dba0ae87c9d43526a5dd6fd26bd2a664483d _7061961_copy.ORF
+f602f8ddeeb9b062f60bfdd3ba0d560c8e54f1f64dbc20e18df5222e2f37a428 DSC00009.ARW
diff --git a/xtask/src/corpus.rs b/xtask/src/corpus.rs
index 67fa842615..84754a0391 100644
--- a/xtask/src/corpus.rs
+++ b/xtask/src/corpus.rs
@@ -10,15 +10,26 @@ use crate::{cargo, root, run};
/// Crates with corpus tests (behind their `corpus` feature).
pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine"];
+/// Opt-in corpus crates: fetched and tested only with `test-corpus --pixls`.
+pub const PIXLS_CRATES: &[&str] = &["photocraft-raw"];
/// Corpus crates with a `heif` feature: test-corpus enables it so the HEIF corpus tests run.
const HEIF_CRATES: &[&str] = &["photocraft-codecs", "photocraft-io"];
/// Paths whose changes make `test-corpus --changed` run (the file-format and rendering crates).
-const CRITICAL: &[&str] =
- &["crates/psd/", "crates/io/", "crates/codecs/", "crates/heif/", "crates/compose/", "crates/gpu/", "crates/text/", "crates/format/", "crates/affinity/"];
-
-/// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --affinity | --photoshop [--local]] [--update-manifest]`
+const CRITICAL: &[&str] = &[
+ "crates/psd/",
+ "crates/io/",
+ "crates/codecs/",
+ "crates/heif/",
+ "crates/compose/",
+ "crates/gpu/",
+ "crates/text/",
+ "crates/format/",
+ "crates/affinity/",
+ "crates/raw/",
+];
+/// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --pixls | --affinity | --photoshop [--local]] [--update-manifest]`
pub fn cmd(args: &[&str]) -> Result<(), String> {
let update = args.contains(&"--update-manifest");
let mut did = false;
@@ -41,6 +52,10 @@ pub fn cmd(args: &[&str]) -> Result<(), String> {
corpus_pins::HEIF.fetch(update)?;
did = true;
}
+ if args.contains(&"--pixls") {
+ crate::pinned::fetch_pixls(update)?;
+ did = true;
+ }
if args.contains(&"--exr") {
corpus_pins::EXR.fetch(update)?;
did = true;
@@ -119,7 +134,8 @@ fn list() {
corpus/affinity/ [{}] 21 public Affinity 1–3 documents (CC0, MIT): vector-art, AFDesignLoad,
Jac21/Branding, AssetStoreTemplate; manifest xtask/affinity-corpus.sha256. Fetch: --affinity
corpus/pngsuite/ [{}] PngSuite (public domain), {PNGSUITE_URL}. Fetch: --pngsuite
- corpus/tiff/, corpus/raw/ optional, copied in by hand
+ corpus/pixls/ [opt-in] real camera raws from raw.pixls.us (public domain), one per decode
+ path plus the known-unsupported packed ORF. Fetch: --pixls (opt-in)
Pins: xtask/src/corpus_pins.rs. Moving one: change it, then -- --update-manifest.",
corpus.display(),
@@ -180,6 +196,7 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> {
Some(i) => (&args[..i], &args[i + 1..]),
None => (args, &[][..]),
};
+ let mut raw_changed = false;
if ours.contains(&"--changed") {
let changed = changed_files()?;
let hits: Vec<&String> = changed.iter().filter(|f| CRITICAL.iter().any(|c| f.starts_with(c))).collect();
@@ -188,6 +205,7 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> {
return Ok(());
}
println!("test-corpus --changed: {} critical files changed (e.g. {}); running the corpus tests", hits.len(), hits[0]);
+ raw_changed = hits.iter().any(|f| f.starts_with("crates/raw/"));
}
let mut crates: Vec = Vec::new();
let mut it = ours.iter();
@@ -196,18 +214,27 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> {
"-p" | "--package" => {
let name = it.next().ok_or("-p needs a crate name")?;
let full = if name.starts_with("photocraft-") { (*name).to_string() } else { format!("photocraft-{name}") };
- if !CORPUS_CRATES.contains(&full.as_str()) {
- return Err(format!("{full} has no corpus tests (crates: {})", CORPUS_CRATES.join(", ")));
+ if !CORPUS_CRATES.contains(&full.as_str()) && !PIXLS_CRATES.contains(&full.as_str()) {
+ return Err(format!("{full} has no corpus tests (crates: {} or the opt-in {})", CORPUS_CRATES.join(", "), PIXLS_CRATES.join(", ")));
}
crates.push(full);
}
- "--changed" | "--local" => {}
+ "--changed" | "--local" | "--pixls" => {}
other => return Err(format!("test-corpus: unknown argument `{other}`")),
}
}
if crates.is_empty() {
crates = CORPUS_CRATES.iter().map(|s| (*s).to_string()).collect();
}
+ // `--pixls` opts in (and fetches), and a raw change under --changed does too: the raw
+ // corpus tests are not part of the default set until a maintainer decides otherwise.
+ let wants_pixls = ours.contains(&"--pixls") || crates.iter().any(|c| PIXLS_CRATES.contains(&c.as_str())) || raw_changed;
+ if wants_pixls && !crates.iter().any(|c| c == "photocraft-raw") {
+ crates.push("photocraft-raw".to_string());
+ }
+ if wants_pixls {
+ crate::pinned::fetch_pixls(false)?;
+ }
fetch_all(ours.contains(&"--local"))?;
let mut c = cargo();
c.args(["test", "--release", "--lib", "--tests"]);
diff --git a/xtask/src/corpus_pins.rs b/xtask/src/corpus_pins.rs
index 8e12c69197..ef7bebe047 100644
--- a/xtask/src/corpus_pins.rs
+++ b/xtask/src/corpus_pins.rs
@@ -224,6 +224,22 @@ pub const AFFINITY: PinnedCorpus = PinnedCorpus {
)
},
};
-
/// Every pinned corpus, in fetch order.
pub const ALL: &[&PinnedCorpus] = &[&PSD_MIXED, &PSD_TOOLS, &PHOTOSHOP, &HEIF, &EXR, &AFFINITY];
+
+/// `corpus/pixls`: real camera raws from (released into the public
+/// domain / CC0 by their photographers; see ATTRIBUTION.md). One file per decode path
+/// `photocraft-raw` supports, plus the packed-ORF known-unsupported oracle. Fetched as single
+/// files (the server 301-redirects `/data/` to `/download/`), verified against
+/// `xtask/pixls.sha256`.
+pub const PIXLS_BASE: &str = "https://raw.pixls.us/data/";
+/// (url path, file name in `corpus/pixls`)
+pub const PIXLS_FILES: &[(&str, &str)] = &[
+ ("Canon/PowerShot%20SX50%20HS/IMG_4059.CR2", "IMG_4059.CR2"),
+ ("Canon/PowerShot%20SX50%20HS/CRW_4061.DNG", "CRW_4061.DNG"),
+ ("Nikon/D3/JD1_8203.NEF", "JD1_8203.NEF"),
+ ("Sony/DSC-RX0/DSC00009.ARW", "DSC00009.ARW"),
+ ("Panasonic/DC-G9/P1000475.RW2", "P1000475.RW2"),
+ ("Olympus/E-1/E_1__C106743_gredos.ORF", "E_1__C106743_gredos.ORF"),
+ ("Olympus/E-5/_7061961_copy.ORF", "_7061961_copy.ORF"),
+];
diff --git a/xtask/src/main.rs b/xtask/src/main.rs
index 634a6aa565..50f3b6b13b 100644
--- a/xtask/src/main.rs
+++ b/xtask/src/main.rs
@@ -25,13 +25,14 @@ commands:
layers enforce the crate dependency layering (plan/architecture.md §3)
wasm cargo check --target wasm32-unknown-unknown for the wasm-safe crates
ci fmt --check, clippy -D warnings, test, layers, wasm (stops at first failure)
- corpus [--all | --pngsuite | --psd | --psd-tools | --photoshop] [--local] [--update-manifest]
+ corpus [--all | --pngsuite | --psd | --psd-tools | --pixls | --photoshop] [--local] [--update-manifest]
show where test corpora live and their pins (xtask/src/corpus_pins.rs), or fetch
- them into corpus/ (pinned commits, sha256-verified; --all = every corpus;
+ them into corpus/ (pinned commits, sha256-verified; --all = every corpus but pixls,
--photoshop --local copies from ../photocraft-corpus or $PHOTOCRAFT_CORPUS_REPO)
- test-corpus [-p ]... [--changed] [--local] [-- ]
+ test-corpus [-p ]... [--changed] [--pixls] [--local] [-- ]
fetch every corpus, then cargo test --release --features corpus on the corpus
- crates; --changed runs only if psd/io/codecs/compose/gpu/text/format changed;
+ crates; --changed runs only if psd/io/codecs/compose/gpu/text/format/raw changed;
+ --pixls (or -p raw) also fetches corpus/pixls and runs the raw corpus tests;
--local takes corpus/photoshop from the photocraft-corpus authoring clone
stats [--exact] count tests and lines per crate (--exact: ask the test harness via `-- --list`)
parity Photoshop menu parity; rewrites docs/parity.md
diff --git a/xtask/src/pinned.rs b/xtask/src/pinned.rs
index 130e19d345..917c0b864b 100644
--- a/xtask/src/pinned.rs
+++ b/xtask/src/pinned.rs
@@ -263,3 +263,52 @@ impl PinnedCorpus {
self.install(&found, |f| Some(src.join(&f[up.prefix.len()..])), &extras, true, &format!("copied from the local clone {}", clone.display()))
}
}
+
+/// Hex sha256 of a corpus file (reused by every corpus fetch, `xtask/pixls.sha256` included).
+fn file_hex(path: &std::path::Path) -> Option {
+ std::fs::read(path).ok().map(|b| crate::sha256::hex(&b))
+}
+
+/// Fetches `corpus/pixls` — the raw.pixls.us camera files pinned in `corpus_pins::PIXLS_FILES` —
+/// and verifies them against `xtask/pixls.sha256`, the same manifest form every corpus uses.
+/// raw.pixls.us serves single files and redirects `/data/` to `/download/`, so this is a plain
+/// per-file download rather than a git upstream; the manifest discipline is the same
+/// (`--update-manifest` re-pins). Opt-in: not part of `--all` or CI until a maintainer says so.
+pub fn fetch_pixls(update: bool) -> Result<(), String> {
+ let dest = crate::root().join("corpus").join("pixls");
+ std::fs::create_dir_all(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
+ let manifest = crate::root().join("xtask").join("pixls.sha256");
+ if update {
+ let mut names: Vec<&str> = crate::corpus_pins::PIXLS_FILES.iter().map(|(_, n)| *n).collect();
+ names.sort_unstable();
+ let mut out = String::new();
+ for name in names {
+ let sum = file_hex(&dest.join(name)).ok_or_else(|| format!("pixls {name}: not downloaded yet"))?;
+ out.push_str(&format!("{sum} {name}\n"));
+ }
+ std::fs::write(&manifest, out).map_err(|e| format!("write {}: {e}", manifest.display()))?;
+ println!("pixls: re-hashed {} files into {}", crate::corpus_pins::PIXLS_FILES.len(), manifest.display());
+ return Ok(());
+ }
+ let want: std::collections::HashMap = std::fs::read_to_string(&manifest)
+ .map_err(|e| format!("read {}: {e} (run `cargo xtask corpus --pixls` once)", manifest.display()))?
+ .lines()
+ .filter_map(|l| l.split_once(" ").map(|(s, n)| (n.to_string(), s.to_string())))
+ .collect();
+ for (path, name) in crate::corpus_pins::PIXLS_FILES {
+ let file = dest.join(name);
+ if file_hex(&file).as_deref() == want.get(*name).map(String::as_str) {
+ continue;
+ }
+ let url = format!("{}{}", crate::corpus_pins::PIXLS_BASE, path);
+ let mut curl = Command::new("curl");
+ curl.args(["-fsSL", "--retry", "3", "-A", USER_AGENT, "-o"]).arg(&file).arg(&url);
+ run(curl, &format!("curl {url}"))?;
+ let got = file_hex(&file).ok_or_else(|| format!("pixls {name}: unreadable"))?;
+ if Some(got.as_str()) != want.get(*name).map(String::as_str) {
+ return Err(format!("pixls {name}: sha256 mismatch (pin the file you meant, then --update-manifest)"));
+ }
+ }
+ println!("pixls: {} files verified in {}", crate::corpus_pins::PIXLS_FILES.len(), dest.display());
+ Ok(())
+}