diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md
index 303d5b821b..4b1a1ff324 100644
--- a/ATTRIBUTION.md
+++ b/ATTRIBUTION.md
@@ -85,7 +85,7 @@ next to the files:
| `corpus/heif/` (9 files, 0.1 MB) | HEIC/HEIF test files: `heic-rs/` checkerboards, RGB strips and a grid-tiled photo with EXIF and XMP (synthetic pixels encoded by macOS `sips`, each `.ref.png` Apple's decode); `pillow-heif/` the 10-bit RGBA `RGBA_10__29x100.heif` and its source `RGBA_16__29x100.png` | Thomas Braun (heic-rs); Pillow-Heif contributors | [heic-rs `tests/fixtures`](https://github.com/tbraun96/heic-rs/tree/main/tests/fixtures), [pillow-heif `tests/images`](https://github.com/bigcat88/pillow_heif/tree/master/tests/images) | MIT OR Apache-2.0 (heic-rs); BSD-3-Clause (pillow-heif) |
| `corpus/exr/` (5 files, 2.3 MB) | The deep OpenEXR test images `11`, `42`, `64`, `multivariate` and `objectid.deep.exr` | Contributors to the OpenEXR Project | [openexr `src/test/bin/test_images`](https://github.com/AcademySoftwareFoundation/openexr/tree/main/src/test/bin/test_images) | BSD-3-Clause |
| `corpus/affinity/` (21 files, 2.4 MB) | Public Affinity documents, unmodified: `vector-art/` (9), `afdesignload/` (9), `jac21/` (2) and `asset-store-template/` (1), each with its upstream licence next to it | samuel-etver; Nick Beeuwsaert; Jeremy Cantu; DAIKI (eviltwo) | [vector-art `simple/`](https://github.com/samuel-etver/vector-art/tree/255f8add3c8f0740196e22bd59502b811b532f0b/simple), [AFDesignLoad `testDesigns/`](https://github.com/NickBeeuwsaert/AFDesignLoad/tree/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns), [Jac21/Branding `Logos/JC/DesignerFiles/Affinity`](https://github.com/Jac21/Branding/tree/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity), [AssetStoreTemplate `AssetStoreTemplate/`](https://github.com/eviltwo/AssetStoreTemplate/tree/f671981ee89d15526285b3cc87b4393d3defe644/AssetStoreTemplate) | CC0-1.0 (vector-art); MIT, Copyright (c) 2015 Nick Beeuwsaert; MIT, Copyright (c) 2018 Jeremy Cantu; MIT, Copyright (c) 2024 DAIKI |
-| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case | The photographers of the pixls.us raw samples database (per-file credits there) | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain (per the database's release) |
+| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case. Each file (and its licence, per the database) is reachable at its path under [raw.pixls.us/data](https://raw.pixls.us/data/): [Canon SX50 HS](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/), [Nikon D3](https://raw.pixls.us/data/Nikon/D3/), [Sony DSC-RX0](https://raw.pixls.us/data/Sony/DSC-RX0/), [Panasonic DC-G9](https://raw.pixls.us/data/Panasonic/DC-G9/), [Olympus E-1](https://raw.pixls.us/data/Olympus/E-1/), [Olympus E-5](https://raw.pixls.us/data/Olympus/E-5/) | The photographers of the pixls.us raw samples database (per-file credits there) | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain, per file as shown in the [database listing](https://raw.pixls.us) |
| `corpus/pngsuite/` | PngSuite | Willem van Schaik | | Public domain |
Files copied into `corpus/` by hand (tiff, exr, raw) must be MIT, BSD or CC0.
diff --git a/Cargo.lock b/Cargo.lock
index 9fd6dee06b..9dd34b93c9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -630,7 +630,7 @@ dependencies = [
"cc",
"cfg-if",
"constant_time_eq",
- "cpufeatures 0.3.1",
+ "cpufeatures",
]
[[package]]
@@ -639,15 +639,6 @@ version = "0.1.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a"
-[[package]]
-name = "block-buffer"
-version = "0.10.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
-dependencies = [
- "generic-array",
-]
-
[[package]]
name = "block2"
version = "0.5.1"
@@ -869,7 +860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
dependencies = [
"cfg-if",
- "cpufeatures 0.3.1",
+ "cpufeatures",
"rand_core 0.10.1",
]
@@ -1096,15 +1087,6 @@ dependencies = [
"libm",
]
-[[package]]
-name = "cpufeatures"
-version = "0.2.17"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
-dependencies = [
- "libc",
-]
-
[[package]]
name = "cpufeatures"
version = "0.3.1"
@@ -1197,16 +1179,6 @@ version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
-[[package]]
-name = "crypto-common"
-version = "0.1.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
-dependencies = [
- "generic-array",
- "typenum",
-]
-
[[package]]
name = "cursor-icon"
version = "1.2.0"
@@ -1266,16 +1238,6 @@ dependencies = [
"rayon",
]
-[[package]]
-name = "digest"
-version = "0.10.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
-dependencies = [
- "block-buffer",
- "crypto-common",
-]
-
[[package]]
name = "dispatch"
version = "0.2.0"
@@ -2068,16 +2030,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "generic-array"
-version = "0.14.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
-dependencies = [
- "typenum",
- "version_check",
-]
-
[[package]]
name = "gethostname"
version = "1.1.0"
@@ -5393,17 +5345,6 @@ dependencies = [
"serde_core",
]
-[[package]]
-name = "sha2"
-version = "0.10.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
-dependencies = [
- "cfg-if",
- "cpufeatures 0.2.17",
- "digest",
-]
-
[[package]]
name = "shlex"
version = "1.3.0"
@@ -6031,12 +5972,6 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c"
-[[package]]
-name = "typenum"
-version = "1.20.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
-
[[package]]
name = "uds_windows"
version = "1.2.1"
@@ -7286,7 +7221,6 @@ version = "0.5.0"
dependencies = [
"serde",
"serde_json",
- "sha2",
"toml",
]
diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml
index 42c86af343..25305d5435 100644
--- a/xtask/Cargo.toml
+++ b/xtask/Cargo.toml
@@ -10,8 +10,6 @@ workspace = true
[dependencies]
serde_json = { workspace = true }
-# corpus manifests (MIT OR Apache-2.0).
-sha2 = "0.10"
serde = { workspace = true }
# perf/budgets.toml and scorecard/*.toml (MIT OR Apache-2.0).
toml = "1"
diff --git a/xtask/src/corpus.rs b/xtask/src/corpus.rs
index cca751559f..424452e0e8 100644
--- a/xtask/src/corpus.rs
+++ b/xtask/src/corpus.rs
@@ -9,7 +9,9 @@ use crate::pinned::USER_AGENT;
use crate::{cargo, root, run};
/// Crates with corpus tests (behind their `corpus` feature).
-pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine", "photocraft-raw"];
+pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine"];
+/// Opt-in corpus crates: fetched and tested only with `test-corpus --pixls`.
+pub const PIXLS_CRATES: &[&str] = &["photocraft-raw"];
/// Corpus crates with a `heif` feature: test-corpus enables it so the HEIF corpus tests run.
const HEIF_CRATES: &[&str] = &["photocraft-codecs", "photocraft-io"];
@@ -27,7 +29,7 @@ const CRITICAL: &[&str] = &[
"crates/affinity/",
"crates/raw/",
];
-/// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --affinity | --photoshop [--local]] [--update-manifest]`
+/// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --pixls | --affinity | --photoshop [--local]] [--update-manifest]`
pub fn cmd(args: &[&str]) -> Result<(), String> {
let update = args.contains(&"--update-manifest");
let mut did = false;
@@ -51,7 +53,7 @@ pub fn cmd(args: &[&str]) -> Result<(), String> {
did = true;
}
if args.contains(&"--pixls") {
- fetch_pixls(update)?;
+ crate::pinned::fetch_pixls(update)?;
did = true;
}
if args.contains(&"--exr") {
@@ -91,7 +93,6 @@ fn local_clone() -> Result {
/// `corpus/photoshop` is copied from the authoring clone of photocraft-corpus instead.
pub fn fetch_all(local: bool) -> Result<(), String> {
fetch_pngsuite()?;
- fetch_pixls(false)?;
for c in corpus_pins::ALL {
if local && std::ptr::eq(*c, &corpus_pins::PHOTOSHOP) {
c.fetch_local(&local_clone()?, false)?;
@@ -210,18 +211,24 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> {
"-p" | "--package" => {
let name = it.next().ok_or("-p needs a crate name")?;
let full = if name.starts_with("photocraft-") { (*name).to_string() } else { format!("photocraft-{name}") };
- if !CORPUS_CRATES.contains(&full.as_str()) {
- return Err(format!("{full} has no corpus tests (crates: {})", CORPUS_CRATES.join(", ")));
+ if !CORPUS_CRATES.contains(&full.as_str()) && !PIXLS_CRATES.contains(&full.as_str()) {
+ return Err(format!("{full} has no corpus tests (crates: {} or the opt-in {})", CORPUS_CRATES.join(", "), PIXLS_CRATES.join(", ")));
}
crates.push(full);
}
- "--changed" | "--local" => {}
+ "--changed" | "--local" | "--pixls" => {}
other => return Err(format!("test-corpus: unknown argument `{other}`")),
}
}
if crates.is_empty() {
crates = CORPUS_CRATES.iter().map(|s| (*s).to_string()).collect();
}
+ // `--pixls` opts in (and fetches): the raw corpus tests are not part of CI until a
+ // maintainer decides they belong there.
+ if ours.contains(&"--pixls") || crates.iter().any(|c| PIXLS_CRATES.contains(&c.as_str())) {
+ crates.push("photocraft-raw".to_string());
+ crate::pinned::fetch_pixls(false)?;
+ }
fetch_all(ours.contains(&"--local"))?;
let mut c = cargo();
c.args(["test", "--release", "--lib", "--tests"]);
@@ -236,55 +243,3 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> {
}
run(c, &format!("cargo test --release --features corpus,heif ({})", crates.join(", ")))
}
-
-/// Fetches `corpus/pixls` (raw.pixls.us single files) and verifies it against
-/// `xtask/pixls.sha256`. With `update`, re-hashes the files instead.
-pub fn fetch_pixls(update: bool) -> Result<(), String> {
- let dest = root().join("corpus").join("pixls");
- std::fs::create_dir_all(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
- let manifest = root().join("xtask").join("pixls.sha256");
- if update {
- let mut out = String::new();
- let mut names: Vec<&str> = corpus_pins::PIXLS_FILES.iter().map(|(_, n)| *n).collect();
- names.sort_unstable();
- for name in names {
- let p = dest.join(name);
- let sum = file_sha256(&p).map_err(|e| format!("{}: {e}", p.display()))?;
- out.push_str(&format!("{sum} {name}\n"));
- }
- std::fs::write(&manifest, out).map_err(|e| format!("write {}: {e}", manifest.display()))?;
- println!("pixls: re-hashed {} files into {}", corpus_pins::PIXLS_FILES.len(), manifest.display());
- return Ok(());
- }
- let want: std::collections::HashMap = std::fs::read_to_string(&manifest)
- .map_err(|e| format!("read {}: {e} (run `cargo xtask corpus --all` once)", manifest.display()))?
- .lines()
- .filter_map(|l| l.split_once(" ").map(|(s, n)| (n.to_string(), s.to_string())))
- .collect();
- for (path, name) in corpus_pins::PIXLS_FILES {
- let file = dest.join(name);
- let have = file_sha256(&file).ok();
- if have.as_deref() == want.get(*name).map(String::as_str) {
- continue;
- }
- let url = format!("{}{}", corpus_pins::PIXLS_BASE, path);
- let mut curl = Command::new("curl");
- curl.args(["-fsSL", "--retry", "3", "-A", USER_AGENT, "-o"]).arg(&file).arg(&url);
- run(curl, &format!("curl {url}"))?;
- let got = file_sha256(&file).map_err(|e| format!("{}: {e}", file.display()))?;
- if Some(got.as_str()) != want.get(*name).map(String::as_str) {
- return Err(format!("pixls {name}: sha256 mismatch (pin the file you meant, then --update-manifest)"));
- }
- }
- println!("pixls: {} files verified in {}", corpus_pins::PIXLS_FILES.len(), dest.display());
- Ok(())
-}
-
-/// Hex sha256 of a file (Windows-friendly: no `sha256sum` dependency).
-fn file_sha256(path: &std::path::Path) -> Result {
- use sha2::{Digest, Sha256};
- let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
- let mut h = Sha256::new();
- h.update(&bytes);
- Ok(h.finalize().into_iter().map(|b| format!("{b:02x}")).collect())
-}
diff --git a/xtask/src/pinned.rs b/xtask/src/pinned.rs
index 130e19d345..917c0b864b 100644
--- a/xtask/src/pinned.rs
+++ b/xtask/src/pinned.rs
@@ -263,3 +263,52 @@ impl PinnedCorpus {
self.install(&found, |f| Some(src.join(&f[up.prefix.len()..])), &extras, true, &format!("copied from the local clone {}", clone.display()))
}
}
+
+/// Hex sha256 of a corpus file (reused by every corpus fetch, `xtask/pixls.sha256` included).
+fn file_hex(path: &std::path::Path) -> Option {
+ std::fs::read(path).ok().map(|b| crate::sha256::hex(&b))
+}
+
+/// Fetches `corpus/pixls` — the raw.pixls.us camera files pinned in `corpus_pins::PIXLS_FILES` —
+/// and verifies them against `xtask/pixls.sha256`, the same manifest form every corpus uses.
+/// raw.pixls.us serves single files and redirects `/data/` to `/download/`, so this is a plain
+/// per-file download rather than a git upstream; the manifest discipline is the same
+/// (`--update-manifest` re-pins). Opt-in: not part of `--all` or CI until a maintainer says so.
+pub fn fetch_pixls(update: bool) -> Result<(), String> {
+ let dest = crate::root().join("corpus").join("pixls");
+ std::fs::create_dir_all(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
+ let manifest = crate::root().join("xtask").join("pixls.sha256");
+ if update {
+ let mut names: Vec<&str> = crate::corpus_pins::PIXLS_FILES.iter().map(|(_, n)| *n).collect();
+ names.sort_unstable();
+ let mut out = String::new();
+ for name in names {
+ let sum = file_hex(&dest.join(name)).ok_or_else(|| format!("pixls {name}: not downloaded yet"))?;
+ out.push_str(&format!("{sum} {name}\n"));
+ }
+ std::fs::write(&manifest, out).map_err(|e| format!("write {}: {e}", manifest.display()))?;
+ println!("pixls: re-hashed {} files into {}", crate::corpus_pins::PIXLS_FILES.len(), manifest.display());
+ return Ok(());
+ }
+ let want: std::collections::HashMap = std::fs::read_to_string(&manifest)
+ .map_err(|e| format!("read {}: {e} (run `cargo xtask corpus --pixls` once)", manifest.display()))?
+ .lines()
+ .filter_map(|l| l.split_once(" ").map(|(s, n)| (n.to_string(), s.to_string())))
+ .collect();
+ for (path, name) in crate::corpus_pins::PIXLS_FILES {
+ let file = dest.join(name);
+ if file_hex(&file).as_deref() == want.get(*name).map(String::as_str) {
+ continue;
+ }
+ let url = format!("{}{}", crate::corpus_pins::PIXLS_BASE, path);
+ let mut curl = Command::new("curl");
+ curl.args(["-fsSL", "--retry", "3", "-A", USER_AGENT, "-o"]).arg(&file).arg(&url);
+ run(curl, &format!("curl {url}"))?;
+ let got = file_hex(&file).ok_or_else(|| format!("pixls {name}: unreadable"))?;
+ if Some(got.as_str()) != want.get(*name).map(String::as_str) {
+ return Err(format!("pixls {name}: sha256 mismatch (pin the file you meant, then --update-manifest)"));
+ }
+ }
+ println!("pixls: {} files verified in {}", crate::corpus_pins::PIXLS_FILES.len(), dest.display());
+ Ok(())
+}