mirror of
https://github.com/storytold/photocraft.git
synced 2026-10-24 09:57:08 -06:00
pixls corpus: in-house sha256 and pinned.rs, opt-in like the affinity corpus
The fetch lives in pinned.rs now, reusing its manifest discipline and the xtask sha256 module (the sha2 dependency is gone). --pixls is opt-in: not part of --all or the CI corpus job (CORPUS_CRATES no longer includes photocraft-raw; test-corpus --pixls fetches and adds it), matching how the Affinity corpus is handled. The usage line names the flag, and the attribution row links each sample's folder in the database so its licence can be checked per file.
This commit is contained in:
Elvis33LE
+1
-1
@@ -85,7 +85,7 @@ next to the files:
|
||||
| `corpus/heif/` (9 files, 0.1 MB) | HEIC/HEIF test files: `heic-rs/` checkerboards, RGB strips and a grid-tiled photo with EXIF and XMP (synthetic pixels encoded by macOS `sips`, each `.ref.png` Apple's decode); `pillow-heif/` the 10-bit RGBA `RGBA_10__29x100.heif` and its source `RGBA_16__29x100.png` | Thomas Braun (heic-rs); Pillow-Heif contributors | [heic-rs `tests/fixtures`](https://github.com/tbraun96/heic-rs/tree/main/tests/fixtures), [pillow-heif `tests/images`](https://github.com/bigcat88/pillow_heif/tree/master/tests/images) | MIT OR Apache-2.0 (heic-rs); BSD-3-Clause (pillow-heif) |
|
||||
| `corpus/exr/` (5 files, 2.3 MB) | The deep OpenEXR test images `11`, `42`, `64`, `multivariate` and `objectid.deep.exr` | Contributors to the OpenEXR Project | [openexr `src/test/bin/test_images`](https://github.com/AcademySoftwareFoundation/openexr/tree/main/src/test/bin/test_images) | BSD-3-Clause |
|
||||
| `corpus/affinity/` (21 files, 2.4 MB) | Public Affinity documents, unmodified: `vector-art/` (9), `afdesignload/` (9), `jac21/` (2) and `asset-store-template/` (1), each with its upstream licence next to it | samuel-etver; Nick Beeuwsaert; Jeremy Cantu; DAIKI (eviltwo) | [vector-art `simple/`](https://github.com/samuel-etver/vector-art/tree/255f8add3c8f0740196e22bd59502b811b532f0b/simple), [AFDesignLoad `testDesigns/`](https://github.com/NickBeeuwsaert/AFDesignLoad/tree/a18dd50a7079fb861a28835eeefa0d015f03f4a1/testDesigns), [Jac21/Branding `Logos/JC/DesignerFiles/Affinity`](https://github.com/Jac21/Branding/tree/57ae3f45bf4637f6927c0b07de25b922c55f944f/Logos/JC/DesignerFiles/Affinity), [AssetStoreTemplate `AssetStoreTemplate/`](https://github.com/eviltwo/AssetStoreTemplate/tree/f671981ee89d15526285b3cc87b4393d3defe644/AssetStoreTemplate) | CC0-1.0 (vector-art); MIT, Copyright (c) 2015 Nick Beeuwsaert; MIT, Copyright (c) 2018 Jeremy Cantu; MIT, Copyright (c) 2024 DAIKI |
|
||||
| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case | The photographers of the pixls.us raw samples database (per-file credits there) | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain (per the database's release) |
|
||||
| `corpus/pixls/` (7 files, 226 MB) | Real camera raws, one per decode path: Canon PowerShot SX50 HS (CR2 and its DNG conversion), Nikon D3 uncompressed NEF, Sony DSC-RX0 cRAW ARW, Panasonic DC-G9 high-res RW2, Olympus E-1 GRBG ORF, plus the E-5 packed ORF as the documented known-unsupported case. Each file (and its licence, per the database) is reachable at its path under [raw.pixls.us/data](https://raw.pixls.us/data/): [Canon SX50 HS](https://raw.pixls.us/data/Canon/PowerShot%20SX50%20HS/), [Nikon D3](https://raw.pixls.us/data/Nikon/D3/), [Sony DSC-RX0](https://raw.pixls.us/data/Sony/DSC-RX0/), [Panasonic DC-G9](https://raw.pixls.us/data/Panasonic/DC-G9/), [Olympus E-1](https://raw.pixls.us/data/Olympus/E-1/), [Olympus E-5](https://raw.pixls.us/data/Olympus/E-5/) | The photographers of the pixls.us raw samples database (per-file credits there) | [raw.pixls.us](https://raw.pixls.us) `data/`, fetched per [xtask/src/corpus_pins.rs](xtask/src/corpus_pins.rs) | CC0 / public domain, per file as shown in the [database listing](https://raw.pixls.us) |
|
||||
| `corpus/pngsuite/` | PngSuite | Willem van Schaik | <http://www.schaik.com/pngsuite/> | Public domain |
|
||||
|
||||
Files copied into `corpus/` by hand (tiff, exr, raw) must be MIT, BSD or CC0.
|
||||
|
||||
Generated
+2
-68
@@ -630,7 +630,7 @@ dependencies = [
|
||||
"cc",
|
||||
"cfg-if",
|
||||
"constant_time_eq",
|
||||
"cpufeatures 0.3.1",
|
||||
"cpufeatures",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -639,15 +639,6 @@ version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a"
|
||||
|
||||
[[package]]
|
||||
name = "block-buffer"
|
||||
version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block2"
|
||||
version = "0.5.1"
|
||||
@@ -869,7 +860,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.3.1",
|
||||
"cpufeatures",
|
||||
"rand_core 0.10.1",
|
||||
]
|
||||
|
||||
@@ -1096,15 +1087,6 @@ dependencies = [
|
||||
"libm",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.2.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cpufeatures"
|
||||
version = "0.3.1"
|
||||
@@ -1197,16 +1179,6 @@ version = "0.2.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5"
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cursor-icon"
|
||||
version = "1.2.0"
|
||||
@@ -1266,16 +1238,6 @@ dependencies = [
|
||||
"rayon",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer",
|
||||
"crypto-common",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dispatch"
|
||||
version = "0.2.0"
|
||||
@@ -2068,16 +2030,6 @@ dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gethostname"
|
||||
version = "1.1.0"
|
||||
@@ -5393,17 +5345,6 @@ dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "sha2"
|
||||
version = "0.10.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"cpufeatures 0.2.17",
|
||||
"digest",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "shlex"
|
||||
version = "1.3.0"
|
||||
@@ -6031,12 +5972,6 @@ version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c"
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.20.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20"
|
||||
|
||||
[[package]]
|
||||
name = "uds_windows"
|
||||
version = "1.2.1"
|
||||
@@ -7286,7 +7221,6 @@ version = "0.5.0"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
"toml",
|
||||
]
|
||||
|
||||
|
||||
@@ -10,8 +10,6 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde_json = { workspace = true }
|
||||
# corpus manifests (MIT OR Apache-2.0).
|
||||
sha2 = "0.10"
|
||||
serde = { workspace = true }
|
||||
# perf/budgets.toml and scorecard/*.toml (MIT OR Apache-2.0).
|
||||
toml = "1"
|
||||
|
||||
+14
-59
@@ -9,7 +9,9 @@ use crate::pinned::USER_AGENT;
|
||||
use crate::{cargo, root, run};
|
||||
|
||||
/// Crates with corpus tests (behind their `corpus` feature).
|
||||
pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine", "photocraft-raw"];
|
||||
pub const CORPUS_CRATES: &[&str] = &["photocraft-psd", "photocraft-codecs", "photocraft-io", "photocraft-engine"];
|
||||
/// Opt-in corpus crates: fetched and tested only with `test-corpus --pixls`.
|
||||
pub const PIXLS_CRATES: &[&str] = &["photocraft-raw"];
|
||||
|
||||
/// Corpus crates with a `heif` feature: test-corpus enables it so the HEIF corpus tests run.
|
||||
const HEIF_CRATES: &[&str] = &["photocraft-codecs", "photocraft-io"];
|
||||
@@ -27,7 +29,7 @@ const CRITICAL: &[&str] = &[
|
||||
"crates/affinity/",
|
||||
"crates/raw/",
|
||||
];
|
||||
/// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --affinity | --photoshop [--local]] [--update-manifest]`
|
||||
/// `cargo xtask corpus [--all | --pngsuite | --download | --psd | --psd-tools | --heif | --exr | --pixls | --affinity | --photoshop [--local]] [--update-manifest]`
|
||||
pub fn cmd(args: &[&str]) -> Result<(), String> {
|
||||
let update = args.contains(&"--update-manifest");
|
||||
let mut did = false;
|
||||
@@ -51,7 +53,7 @@ pub fn cmd(args: &[&str]) -> Result<(), String> {
|
||||
did = true;
|
||||
}
|
||||
if args.contains(&"--pixls") {
|
||||
fetch_pixls(update)?;
|
||||
crate::pinned::fetch_pixls(update)?;
|
||||
did = true;
|
||||
}
|
||||
if args.contains(&"--exr") {
|
||||
@@ -91,7 +93,6 @@ fn local_clone() -> Result<PathBuf, String> {
|
||||
/// `corpus/photoshop` is copied from the authoring clone of photocraft-corpus instead.
|
||||
pub fn fetch_all(local: bool) -> Result<(), String> {
|
||||
fetch_pngsuite()?;
|
||||
fetch_pixls(false)?;
|
||||
for c in corpus_pins::ALL {
|
||||
if local && std::ptr::eq(*c, &corpus_pins::PHOTOSHOP) {
|
||||
c.fetch_local(&local_clone()?, false)?;
|
||||
@@ -210,18 +211,24 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> {
|
||||
"-p" | "--package" => {
|
||||
let name = it.next().ok_or("-p needs a crate name")?;
|
||||
let full = if name.starts_with("photocraft-") { (*name).to_string() } else { format!("photocraft-{name}") };
|
||||
if !CORPUS_CRATES.contains(&full.as_str()) {
|
||||
return Err(format!("{full} has no corpus tests (crates: {})", CORPUS_CRATES.join(", ")));
|
||||
if !CORPUS_CRATES.contains(&full.as_str()) && !PIXLS_CRATES.contains(&full.as_str()) {
|
||||
return Err(format!("{full} has no corpus tests (crates: {} or the opt-in {})", CORPUS_CRATES.join(", "), PIXLS_CRATES.join(", ")));
|
||||
}
|
||||
crates.push(full);
|
||||
}
|
||||
"--changed" | "--local" => {}
|
||||
"--changed" | "--local" | "--pixls" => {}
|
||||
other => return Err(format!("test-corpus: unknown argument `{other}`")),
|
||||
}
|
||||
}
|
||||
if crates.is_empty() {
|
||||
crates = CORPUS_CRATES.iter().map(|s| (*s).to_string()).collect();
|
||||
}
|
||||
// `--pixls` opts in (and fetches): the raw corpus tests are not part of CI until a
|
||||
// maintainer decides they belong there.
|
||||
if ours.contains(&"--pixls") || crates.iter().any(|c| PIXLS_CRATES.contains(&c.as_str())) {
|
||||
crates.push("photocraft-raw".to_string());
|
||||
crate::pinned::fetch_pixls(false)?;
|
||||
}
|
||||
fetch_all(ours.contains(&"--local"))?;
|
||||
let mut c = cargo();
|
||||
c.args(["test", "--release", "--lib", "--tests"]);
|
||||
@@ -236,55 +243,3 @@ pub fn test_cmd(args: &[&str]) -> Result<(), String> {
|
||||
}
|
||||
run(c, &format!("cargo test --release --features corpus,heif ({})", crates.join(", ")))
|
||||
}
|
||||
|
||||
/// Fetches `corpus/pixls` (raw.pixls.us single files) and verifies it against
|
||||
/// `xtask/pixls.sha256`. With `update`, re-hashes the files instead.
|
||||
pub fn fetch_pixls(update: bool) -> Result<(), String> {
|
||||
let dest = root().join("corpus").join("pixls");
|
||||
std::fs::create_dir_all(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||
let manifest = root().join("xtask").join("pixls.sha256");
|
||||
if update {
|
||||
let mut out = String::new();
|
||||
let mut names: Vec<&str> = corpus_pins::PIXLS_FILES.iter().map(|(_, n)| *n).collect();
|
||||
names.sort_unstable();
|
||||
for name in names {
|
||||
let p = dest.join(name);
|
||||
let sum = file_sha256(&p).map_err(|e| format!("{}: {e}", p.display()))?;
|
||||
out.push_str(&format!("{sum} {name}\n"));
|
||||
}
|
||||
std::fs::write(&manifest, out).map_err(|e| format!("write {}: {e}", manifest.display()))?;
|
||||
println!("pixls: re-hashed {} files into {}", corpus_pins::PIXLS_FILES.len(), manifest.display());
|
||||
return Ok(());
|
||||
}
|
||||
let want: std::collections::HashMap<String, String> = std::fs::read_to_string(&manifest)
|
||||
.map_err(|e| format!("read {}: {e} (run `cargo xtask corpus --all` once)", manifest.display()))?
|
||||
.lines()
|
||||
.filter_map(|l| l.split_once(" ").map(|(s, n)| (n.to_string(), s.to_string())))
|
||||
.collect();
|
||||
for (path, name) in corpus_pins::PIXLS_FILES {
|
||||
let file = dest.join(name);
|
||||
let have = file_sha256(&file).ok();
|
||||
if have.as_deref() == want.get(*name).map(String::as_str) {
|
||||
continue;
|
||||
}
|
||||
let url = format!("{}{}", corpus_pins::PIXLS_BASE, path);
|
||||
let mut curl = Command::new("curl");
|
||||
curl.args(["-fsSL", "--retry", "3", "-A", USER_AGENT, "-o"]).arg(&file).arg(&url);
|
||||
run(curl, &format!("curl {url}"))?;
|
||||
let got = file_sha256(&file).map_err(|e| format!("{}: {e}", file.display()))?;
|
||||
if Some(got.as_str()) != want.get(*name).map(String::as_str) {
|
||||
return Err(format!("pixls {name}: sha256 mismatch (pin the file you meant, then --update-manifest)"));
|
||||
}
|
||||
}
|
||||
println!("pixls: {} files verified in {}", corpus_pins::PIXLS_FILES.len(), dest.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Hex sha256 of a file (Windows-friendly: no `sha256sum` dependency).
|
||||
fn file_sha256(path: &std::path::Path) -> Result<String, String> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let bytes = std::fs::read(path).map_err(|e| e.to_string())?;
|
||||
let mut h = Sha256::new();
|
||||
h.update(&bytes);
|
||||
Ok(h.finalize().into_iter().map(|b| format!("{b:02x}")).collect())
|
||||
}
|
||||
|
||||
@@ -263,3 +263,52 @@ impl PinnedCorpus {
|
||||
self.install(&found, |f| Some(src.join(&f[up.prefix.len()..])), &extras, true, &format!("copied from the local clone {}", clone.display()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Hex sha256 of a corpus file (reused by every corpus fetch, `xtask/pixls.sha256` included).
|
||||
fn file_hex(path: &std::path::Path) -> Option<String> {
|
||||
std::fs::read(path).ok().map(|b| crate::sha256::hex(&b))
|
||||
}
|
||||
|
||||
/// Fetches `corpus/pixls` — the raw.pixls.us camera files pinned in `corpus_pins::PIXLS_FILES` —
|
||||
/// and verifies them against `xtask/pixls.sha256`, the same manifest form every corpus uses.
|
||||
/// raw.pixls.us serves single files and redirects `/data/` to `/download/`, so this is a plain
|
||||
/// per-file download rather than a git upstream; the manifest discipline is the same
|
||||
/// (`--update-manifest` re-pins). Opt-in: not part of `--all` or CI until a maintainer says so.
|
||||
pub fn fetch_pixls(update: bool) -> Result<(), String> {
|
||||
let dest = crate::root().join("corpus").join("pixls");
|
||||
std::fs::create_dir_all(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||
let manifest = crate::root().join("xtask").join("pixls.sha256");
|
||||
if update {
|
||||
let mut names: Vec<&str> = crate::corpus_pins::PIXLS_FILES.iter().map(|(_, n)| *n).collect();
|
||||
names.sort_unstable();
|
||||
let mut out = String::new();
|
||||
for name in names {
|
||||
let sum = file_hex(&dest.join(name)).ok_or_else(|| format!("pixls {name}: not downloaded yet"))?;
|
||||
out.push_str(&format!("{sum} {name}\n"));
|
||||
}
|
||||
std::fs::write(&manifest, out).map_err(|e| format!("write {}: {e}", manifest.display()))?;
|
||||
println!("pixls: re-hashed {} files into {}", crate::corpus_pins::PIXLS_FILES.len(), manifest.display());
|
||||
return Ok(());
|
||||
}
|
||||
let want: std::collections::HashMap<String, String> = std::fs::read_to_string(&manifest)
|
||||
.map_err(|e| format!("read {}: {e} (run `cargo xtask corpus --pixls` once)", manifest.display()))?
|
||||
.lines()
|
||||
.filter_map(|l| l.split_once(" ").map(|(s, n)| (n.to_string(), s.to_string())))
|
||||
.collect();
|
||||
for (path, name) in crate::corpus_pins::PIXLS_FILES {
|
||||
let file = dest.join(name);
|
||||
if file_hex(&file).as_deref() == want.get(*name).map(String::as_str) {
|
||||
continue;
|
||||
}
|
||||
let url = format!("{}{}", crate::corpus_pins::PIXLS_BASE, path);
|
||||
let mut curl = Command::new("curl");
|
||||
curl.args(["-fsSL", "--retry", "3", "-A", USER_AGENT, "-o"]).arg(&file).arg(&url);
|
||||
run(curl, &format!("curl {url}"))?;
|
||||
let got = file_hex(&file).ok_or_else(|| format!("pixls {name}: unreadable"))?;
|
||||
if Some(got.as_str()) != want.get(*name).map(String::as_str) {
|
||||
return Err(format!("pixls {name}: sha256 mismatch (pin the file you meant, then --update-manifest)"));
|
||||
}
|
||||
}
|
||||
println!("pixls: {} files verified in {}", crate::corpus_pins::PIXLS_FILES.len(), dest.display());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user