mirror of
https://github.com/storytold/lightcraft.git
synced 2026-10-24 10:09:01 -06:00
Merge pull request #546 from storytold/claude/integrate-batch-19
Integrate #474, #448, #447, #499, #459
This commit is contained in:
Brandon Thomas
Generated
+1
@@ -6824,6 +6824,7 @@ dependencies = [
|
||||
"brotli",
|
||||
"flate2",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -305,7 +305,7 @@ LightCraft is young and moving fast. **Where we honestly stand** (details in the
|
||||
| Profiles (Color, Neutral, Vivid, Landscape, Portrait, Monochrome: our own looks), presets, versions, history, copy/paste/sync settings | ✅ |
|
||||
| Camera colour: DNG files use their own matrices | ✅ DNG · 🟡 own Sony/Fujifilm profiles; measured calibration database missing |
|
||||
| Native macOS menu bar (generated from the command registry), control channel + every widget addressable, headless UI snapshots | ✅ |
|
||||
| RAW: DNG, CR2, CR3 (lossless CRX Bayer and version 0x100/0x200 C-RAW), ARW, NEF (uncompressed + lossless/lossy compressed), Fujifilm RAF (uncompressed + lossless/lossy compressed, Bayer + X-Trans), Panasonic RW2 / Leica RWL / Panasonic RAW (every raw format, DMC-LX1 to DC-S1RM2), Pentax PEF, Olympus ORF (uncompressed); embedded previews for every format incl. CR3 | 🟡 · CR3, compressed ORF decode ⬜ |
|
||||
| RAW: DNG, CR2, CR3 (lossless CRX Bayer and version 0x100/0x200 C-RAW), ARW, NEF (uncompressed + lossless/lossy compressed), Fujifilm RAF (uncompressed + lossless/lossy compressed, Bayer + X-Trans), Panasonic RW2 / Leica RWL / Panasonic RAW (every raw format, DMC-LX1 to DC-S1RM2), Pentax PEF, Olympus ORF (uncompressed), Samsung SRW (uncompressed); embedded previews for every format incl. CR3 | 🟡 · CR3, compressed ORF and compressed SRW decode ⬜ |
|
||||
| Detail: sharpening, luminance + colour noise reduction | ✅ · AI Denoise, Super Resolution ⬜ |
|
||||
| Remove / Heal / Clone spots (auto source), Visualize Spots, Red Eye and Pet Eye (auto pupil detection, catchlight) | ✅ · content-aware fill, spot pin editing 🚧 |
|
||||
| Export: JPEG / PNG / TIFF / WebP / AVIF / DNG / original, sizing, file-size limit, output sharpening, naming templates, batch, metadata policy, text or image watermark | ✅ · HDR export ⬜ |
|
||||
|
||||
+5
-5
@@ -23,7 +23,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).
|
||||
| Dimension | Estimate | What's true today | Biggest gaps |
|
||||
|---|---:|---|---|
|
||||
| **Feature checklist** | 79.7% | P0 core and P1 nearly complete: import (Add / Copy / Move, templates, devices), library, grid/loupe/compare/survey, every Edit slider, curves, colour grading, masking tools, crop/Upright, heal/clone, presets/profiles, versions/history, sync, export, menus, shortcuts | P0: full-resolution zoom detail; P1: lens-profile database, content-aware fill (patch synthesis), video playback/trim |
|
||||
| **RAW coverage** (formats people shoot) | ~55% | DNG (all kinds), CR2, CR3 lossless Bayer and version 0x100/0x200 C-RAW (M50/R100/R8 pixel-exact corpus), ARW, NEF (uncompressed + Huffman lossless/lossy), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans), uncompressed ORF, RW2 / RWL / Panasonic RAW (every raw format, checked on 178 files from 118 Panasonic and Leica bodies), PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed ORF, Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. NEFs labelled compressed but stored uncompressed (Z 6 packed 14-bit, D850 12-bit uncompressed). Per-model verification is thin (~55 corpus files vs >1,000 models) |
|
||||
| **RAW coverage** (formats people shoot) | ~55% | DNG (all kinds), CR2, CR3 lossless Bayer and version 0x100/0x200 C-RAW (M50/R100/R8 pixel-exact corpus), ARW, NEF (uncompressed + Huffman lossless/lossy), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans), uncompressed ORF, uncompressed Samsung SRW (NX5 to NX1100, EX1, WB2000), RW2 / RWL / Panasonic RAW (every raw format, checked on 178 files from 118 Panasonic and Leica bodies), PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed ORF, compressed Samsung SRW (NX1, NX30, NX300, NX500, NX2000, NX3000, NX3300, NX mini), Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. NEFs labelled compressed but stored uncompressed (Z 6 packed 14-bit, D850 12-bit uncompressed). Per-model verification is thin (~55 corpus files vs >1,000 models) |
|
||||
| **Colour & image quality** | ~55–65% | Pipeline is complete and fast; GPU path CPU-exact within 1/255 | **No measured camera calibration database**: ARW, NEF, RW2, RAF, CR3, CR2 and PEF have guarded camera-JPEG colour estimates, with bundled ILCE-7M4, X-H2S and X-T4 profiles (docs/camera-preview-colour.md); other non-DNG raws and rejected estimates use a neutral matrix. Colour fidelity remains incomplete. No lens-profile database. No measured fidelity against Lightroom (tone, highlights, texture/clarity, NR, sharpening are tuned by eye) |
|
||||
| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics, or the photo's own segmentation mattes when it ships them (DNG semantic masks, e.g. iPhone ProRAW's sky matte); non-destructive Bayer RAW denoise with opt-in weights on pure-Rust CPU/GPU; **faces (MVP, [docs/faces.md](docs/faces.md))**: a pure-Rust detector (a 232 KB opt-in download), opt-in recognition models (one-click download or add a file; they run on the shared pure-Rust CPU interpreter; pinned SFace/AuraFace embeddings agree with an independent reference runtime on 40 public crops; one-thread medians 34–36 / 233–252 ms (AuraFace remains expensive)), a background scan that finds and embeds faces, name suggestions, a person's page with look-alike faces to confirm, an Unnamed faces section (look-alikes together, select a group and name it at once) | Real segmentation masks (subject, sky, people, objects, landscape, depth), denoise model policy / linear RGB / quality verification, super resolution, lens blur, generative remove, natural-language search; for faces: clusters with separators between groups, small faces in large photos (the detector looks at 640 px), thresholds calibrated on live faces, writing face regions back to XMP. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) |
|
||||
| **Workflow & library** | ~85% (single machine) | Robust catalog (journal + snapshots, background compaction, crash-tested), 85k-photo libraries stay responsive, Local browsing with automatic cleanup, XMP interop, keywords, smart albums, Move import | Opening an 85k library takes 1.7–4.7 s; no cloud sync (out of scope), no tablet companion (#74, roadmap), shared albums, publish services, tethering |
|
||||
@@ -47,8 +47,8 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).
|
||||
Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/parity.md) → *Top gaps*.
|
||||
|
||||
1. **Camera colour calibration of our own** (LR-PROF-CAMERACOLOR, P0): fit each camera to its own embedded JPEG, use
|
||||
matrices the files carry themselves, then chart shots. Sony ARW, Nikon NEF, Panasonic RW2, Fujifilm RAF, Canon CR3/CR2
|
||||
and Pentax PEF have the file-local fit (matrix + tone curve from their own JPEG; CR2 and PEF since issue #310, not yet
|
||||
matrices the files carry themselves, then chart shots. Sony ARW, Nikon NEF, Panasonic RW2, Fujifilm RAF, Canon CR3/CR2,
|
||||
Pentax PEF and Samsung SRW have the file-local fit (matrix + tone curve from their own JPEG; CR2 and PEF since issue #310, not yet
|
||||
checked on the corpus); then ORF, then validate fidelity.
|
||||
2. **Raw formats, clean-room** (LR-IMP-FORMATS, P0): Remaining **unverified CR3 CRX** variants, compressed ORF, NEF
|
||||
lossy-after-split, sRAW. Decided 2026-10-05: write our own decoders from prose descriptions (never decoder source,
|
||||
@@ -175,7 +175,7 @@ CR3 lossless Bayer and version 0x100/0x200 C-RAW sensor decoding is verified pix
|
||||
|
||||
Decoded (CC0 corpus from raw.pixls.us, `cargo xtask corpus --download`, `crates/raw/tests/corpus.rs`): DNG (uncompressed,
|
||||
LJ92, lossy JPEG / Smart Previews, Deflate, float, linear, DNG 1.7 JPEG XL tiles: lossless ones bit-exact on synthetic files,
|
||||
lossy ones decode but no real file has verified them), CR2, ARW (uncompressed, ARW2, LJ92; as-shot white balance and black level of pre-2017 bodies from the enciphered
|
||||
lossy ones unclipped in the declared linear primaries, checked on one real file), CR2, ARW (uncompressed, ARW2, LJ92; as-shot white balance and black level of pre-2017 bodies from the enciphered
|
||||
maker-note `Tag2010` and the encrypted `SR2SubIFD`, both recovered by black-box analysis, `crates/raw/src/vendor/arw.rs`), NEF/NRW uncompressed and Huffman-compressed (lossless, lossy type 1/2, 12/14-bit), RAF uncompressed and lossless/lossy compressed (Bayer and
|
||||
X-Trans, 14/16-bit real-file verification), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed formats 4 and 6, the prefix-coded strips of format 8,
|
||||
packed formats 2, 5 and 7, the 16-bit words of the 2005–2007 bodies; all recovered by black-box analysis of 178 CC0 files from
|
||||
@@ -188,7 +188,7 @@ Not decoded yet — preview only (no permissively licensed description; black-bo
|
||||
alignment or reset predictors at the split row). The other Nikon Huffman variants are decoded (`crates/raw/src/vendor/nefc.rs` documents the analysis).
|
||||
- **Olympus compressed ORF**, **Canon CR3 unverified CRX variants** (M11.1), **Canon sRAW/mRAW**.
|
||||
|
||||
**Camera colour matrices:** ARW, NEF, RW2, RAF, CR3, CR2 and PEF files can use guarded, separate chromaticity and tone estimates from their own embedded JPEG (see `docs/camera-preview-colour.md`); this is a per-file camera-look estimate with relative WB, not measured calibration or absolute-Kelvin WB. Other non-DNG raws and rejected fits use the documented neutral fallback (camera RGB ≈ linear sRGB, flagged
|
||||
**Camera colour matrices:** ARW, NEF, RW2, RAF, CR3, CR2, PEF and Samsung SRW files can use guarded, separate chromaticity and tone estimates from their own embedded JPEG (see `docs/camera-preview-colour.md`); this is a per-file camera-look estimate with relative WB, not measured calibration or absolute-Kelvin WB. Other non-DNG raws and rejected fits use the documented neutral fallback (camera RGB ≈ linear sRGB, flagged
|
||||
`matrix_is_fallback`) with the file's as-shot white-balance multipliers. Clean sources to evaluate next: manufacturer
|
||||
matrices stored in the files themselves (Olympus ImageProcessing `ColorMatrix`, Pentax equivalents; Panasonic RW2 files carry none) and our
|
||||
own chart-based calibration (M11.4). Adobe matrices are never used.
|
||||
|
||||
@@ -336,13 +336,13 @@ impl Photo {
|
||||
pub fn develops_raw(&self) -> bool {
|
||||
self.kind == MediaKind::Raw && self.preview_only.is_none()
|
||||
}
|
||||
/// The current ARW, NEF, RW2, RAF, CR3, CR2 and PEF readers have vendor WB multipliers but no measured camera
|
||||
/// The current ARW, NEF, RW2, RAF, CR3, CR2, PEF and SRW readers have vendor WB multipliers but no measured camera
|
||||
/// illuminant. Use adjustments relative to the camera's as-shot look, as for rendered
|
||||
/// photographs (the engine's `camera_preview::file_local_look` covers the same formats; RWL and
|
||||
/// RAW are Leica's and the oldest Panasonic bodies' names for RW2 files).
|
||||
pub fn relative_wb(&self) -> bool {
|
||||
self.develops_raw()
|
||||
&& ["ARW", "NEF", "NRW", "RW2", "RWL", "RAW", "RAF", "CR3", "CR2", "PEF"].iter().any(|f| self.format.eq_ignore_ascii_case(f))
|
||||
&& ["ARW", "NEF", "NRW", "RW2", "RWL", "RAW", "RAF", "CR3", "CR2", "PEF", "SRW"].iter().any(|f| self.format.eq_ignore_ascii_case(f))
|
||||
}
|
||||
/// The develop settings import gave this photo: [`Photo::camera_defaults`], or the user's
|
||||
/// default preset applied on top of them ([`Photo::import_look`]).
|
||||
@@ -450,6 +450,7 @@ mod edited_tests {
|
||||
("a.raw", "RAW", true),
|
||||
("a.raf", "rAf", true),
|
||||
("a.cr3", "cr3", true),
|
||||
("a.srw", "SRW", true),
|
||||
("a.dng", "DNG", false),
|
||||
] {
|
||||
let mut p = Photo::new(PhotoId(1), Source::Demo { scene: 0 }, name, format, 10, 10, "2026-10-01T00:00:00");
|
||||
|
||||
@@ -33,7 +33,15 @@ pub(crate) const PROFILE_PROXY: usize = 192;
|
||||
pub(crate) fn file_local_look(format: RawFormat) -> bool {
|
||||
matches!(
|
||||
format,
|
||||
RawFormat::Arw | RawFormat::Nef | RawFormat::Nrw | RawFormat::Rw2 | RawFormat::Raf | RawFormat::Cr3 | RawFormat::Cr2 | RawFormat::Pef
|
||||
RawFormat::Arw
|
||||
| RawFormat::Nef
|
||||
| RawFormat::Nrw
|
||||
| RawFormat::Rw2
|
||||
| RawFormat::Raf
|
||||
| RawFormat::Cr3
|
||||
| RawFormat::Cr2
|
||||
| RawFormat::Pef
|
||||
| RawFormat::Srw
|
||||
)
|
||||
}
|
||||
|
||||
@@ -402,19 +410,38 @@ const EDGE_CONTRAST: f32 = 3.0;
|
||||
/// it is tried once more on the pixels away from edges (same gates): there colour pairs stay
|
||||
/// valid when the camera JPEG's geometry differs slightly from the raw's.
|
||||
fn fit_pairs_with(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>) -> Option<CameraLook> {
|
||||
fit_pairs_ordered(sensor, reference, colour, &[ToneFit::Quantile, ToneFit::ConditionalMedian])
|
||||
}
|
||||
|
||||
/// [`fit_pairs_with`] with the tone fits to try, in order: the first whose look passes the
|
||||
/// acceptance gates is used. Quantile matching comes first because it cannot produce flat steps;
|
||||
/// the conditional-median fit follows because on a few photos (compacts, early Micro Four Thirds)
|
||||
/// its slightly lower held-out error is what clears the gates, and a photo must not lose its look
|
||||
/// to the change of fit. The whole search (all pixels, then away from edges, with and without the
|
||||
/// camera profile's colour) runs with one tone fit before the next is tried, so the fallback
|
||||
/// accepts exactly what the previous releases accepted.
|
||||
fn fit_pairs_ordered(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, order: &[ToneFit]) -> Option<CameraLook> {
|
||||
let has_profile = colour.is_some();
|
||||
// A different picture style can make a camera profile fail the gates. Preserve the
|
||||
// photo's own colour fit before resorting to the neutral fallback.
|
||||
fit_candidate(sensor, reference, colour).or_else(|| has_profile.then(|| fit_candidate(sensor, reference, None)).flatten())
|
||||
order.iter().find_map(|&tone| {
|
||||
// A different picture style can make a camera profile fail the gates. Preserve the
|
||||
// photo's own colour fit before resorting to the neutral fallback.
|
||||
fit_candidate(sensor, reference, colour.clone(), tone).or_else(|| has_profile.then(|| fit_candidate(sensor, reference, None, tone)).flatten())
|
||||
})
|
||||
}
|
||||
|
||||
/// One candidate colour model, fitted on all pixels and, when that fails the gates, once more on
|
||||
/// the pixels away from edges (issue #232).
|
||||
fn fit_candidate(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>) -> Option<CameraLook> {
|
||||
fit_pairs_on(sensor, reference, colour.clone(), None).or_else(|| fit_pairs_on(sensor, reference, colour, Some(EDGE_CONTRAST)))
|
||||
fn fit_candidate(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, tone: ToneFit) -> Option<CameraLook> {
|
||||
fit_pairs_on(sensor, reference, colour.clone(), None, tone).or_else(|| fit_pairs_on(sensor, reference, colour, Some(EDGE_CONTRAST), tone))
|
||||
}
|
||||
|
||||
fn fit_pairs_on(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Option<HsvTable>)>, edge_limit: Option<f32>) -> Option<CameraLook> {
|
||||
fn fit_pairs_on(
|
||||
sensor: &Rgb32f,
|
||||
reference: &Rgb32f,
|
||||
colour: Option<(Mat3, Option<HsvTable>)>,
|
||||
edge_limit: Option<f32>,
|
||||
tone_fit: ToneFit,
|
||||
) -> Option<CameraLook> {
|
||||
// A known colour model needs enough signal for tone fitting, not a scene rich enough to
|
||||
// learn a new colour matrix. Still reject monochrome references.
|
||||
let (pairs, bright) = collect_pairs(sensor, reference, if colour.is_some() { 0.005 } else { 0.05 }, edge_limit)?;
|
||||
@@ -437,7 +464,7 @@ fn fit_pairs_on(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Optio
|
||||
};
|
||||
let tone_pairs: Vec<_> =
|
||||
pairs.iter().enumerate().filter(|(i, _)| i % 3 != 0).map(|(_, (x, y))| (luma(colour(*x).map(|v| v.max(0.0))), luma(*y))).collect();
|
||||
let Some(curve) = fit_tone(tone_pairs) else { continue };
|
||||
let Some(curve) = tone_fit.fit(tone_pairs) else { continue };
|
||||
let mut look = CameraLook { matrix, tone: curve, hue_sat: hue_sat.clone() };
|
||||
if let Some(tone) = fit_chroma(&bright, &look) {
|
||||
look.tone = tone;
|
||||
@@ -468,10 +495,11 @@ fn fit_pairs_on(sensor: &Rgb32f, reference: &Rgb32f, colour: Option<(Mat3, Optio
|
||||
.sum();
|
||||
if lightcraft_pipeline::profiling() {
|
||||
eprintln!(
|
||||
"[profile] camera look holdout RMS {:.5} -> {:.5} ({samples} channels{})",
|
||||
"[profile] camera look holdout RMS {:.5} -> {:.5} ({samples} channels{}{})",
|
||||
(before / samples as f64).sqrt(),
|
||||
(after / samples as f64).sqrt(),
|
||||
if edge_limit.is_some() { ", away from edges" } else { "" }
|
||||
if edge_limit.is_some() { ", away from edges" } else { "" },
|
||||
if tone_fit == ToneFit::ConditionalMedian { ", conditional-median tone" } else { "" }
|
||||
);
|
||||
}
|
||||
if samples == 0 || after >= before * MIN_IMPROVEMENT || after / samples as f64 > MAX_HOLDOUT_RMS.powi(2) {
|
||||
@@ -681,12 +709,68 @@ fn fit_chroma(pairs: &[([f64; 3], [f64; 3])], look: &CameraLook) -> Option<Camer
|
||||
(after.is_finite() && after < before).then_some(fitted)
|
||||
}
|
||||
|
||||
/// How the per-photo tone curve is fitted ([`fit_pairs_ordered`] tries them in order).
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
enum ToneFit {
|
||||
/// [`fit_tone`]: no flat steps, no pixel-exact registration needed.
|
||||
Quantile,
|
||||
/// [`fit_tone_conditional`]: the earlier fit, kept as the fallback.
|
||||
ConditionalMedian,
|
||||
}
|
||||
|
||||
impl ToneFit {
|
||||
fn fit(self, pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
|
||||
match self {
|
||||
ToneFit::Quantile => fit_tone(pairs),
|
||||
ToneFit::ConditionalMedian => fit_tone_conditional(pairs),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The camera's tone curve from scene/JPEG luminance pairs, by quantile matching: knot `i` pairs
|
||||
/// the median of the `i`-th 1/32 of the sorted scene luminances with the median of the `i`-th
|
||||
/// 1/32 of the sorted JPEG luminances. A tone curve is monotone, so it maps each quantile of the
|
||||
/// scene to the same quantile of the JPEG; matching the two distributions needs no pixel-exact
|
||||
/// registration and is increasing by construction.
|
||||
///
|
||||
/// Binning the pairs by scene luminance and taking the JPEG's median per bin instead (followed by
|
||||
/// isotonic pooling) fails on real files: camera JPEGs are lens-corrected and sharpened, so on a
|
||||
/// 96 px proxy of a busy scene neighbouring pairs disagree by more than the gap between bins.
|
||||
/// The conditional medians then zig-zag, pooling turns them into flat steps (13 of 31 segments
|
||||
/// on a Z 6II forest, issue #475) that render as posterised patches, and regression toward the
|
||||
/// mean flattens both ends of the curve (lifted blacks, dull highlights).
|
||||
fn fit_tone(pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
|
||||
if pairs.len() < 128 || !pairs.iter().all(|(x, y)| x.is_finite() && *x > 0.0 && y.is_finite()) {
|
||||
return None;
|
||||
}
|
||||
let mut xs: Vec<f64> = pairs.iter().map(|p| p.0).collect();
|
||||
let mut ys: Vec<f64> = pairs.iter().map(|p| p.1).collect();
|
||||
xs.sort_by(f64::total_cmp);
|
||||
ys.sort_by(f64::total_cmp);
|
||||
let n = xs.len();
|
||||
let mut knots = [[0.0; 2]; 32];
|
||||
for (i, knot) in knots.iter_mut().enumerate() {
|
||||
let (a, b) = (i * n / 32, (i + 1) * n / 32);
|
||||
let mid = a + (b - a) / 2;
|
||||
*knot = [*xs.get(mid)? as f32, *ys.get(mid)? as f32];
|
||||
}
|
||||
if knots[31][0] < knots[0][0] * 1.5 {
|
||||
return None;
|
||||
}
|
||||
CameraTone::new(knots)
|
||||
}
|
||||
|
||||
fn median(values: &mut [f64]) -> Option<f64> {
|
||||
values.sort_by(f64::total_cmp);
|
||||
values.get(values.len() / 2).copied()
|
||||
}
|
||||
|
||||
fn fit_tone(mut pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
|
||||
/// The earlier tone fit, kept as the fallback of [`ToneFit`]: the JPEG's median per equal-population
|
||||
/// bin of scene luminance, then pooled-adjacent-violators isotonic regression. It follows the
|
||||
/// per-pixel pairs closely when the two images are registered, which is why it can clear the
|
||||
/// acceptance gates where [`fit_tone`] just misses them, but on lens-corrected JPEGs of busy scenes
|
||||
/// the pooling leaves flat steps (issue #475).
|
||||
fn fit_tone_conditional(mut pairs: Vec<(f64, f64)>) -> Option<CameraTone> {
|
||||
if pairs.len() < 128 || !pairs.iter().all(|(x, y)| x.is_finite() && *x > 0.0 && y.is_finite()) {
|
||||
return None;
|
||||
}
|
||||
@@ -1021,7 +1105,17 @@ mod tests {
|
||||
#[test]
|
||||
fn supported_raws_get_a_file_local_look() {
|
||||
// every decoded raw without a colour matrix of its own (issue #310: CR2 rendered flat with the fallback)
|
||||
let local = [RawFormat::Arw, RawFormat::Nef, RawFormat::Nrw, RawFormat::Rw2, RawFormat::Raf, RawFormat::Cr3, RawFormat::Cr2, RawFormat::Pef];
|
||||
let local = [
|
||||
RawFormat::Arw,
|
||||
RawFormat::Nef,
|
||||
RawFormat::Nrw,
|
||||
RawFormat::Rw2,
|
||||
RawFormat::Raf,
|
||||
RawFormat::Cr3,
|
||||
RawFormat::Cr2,
|
||||
RawFormat::Pef,
|
||||
RawFormat::Srw,
|
||||
];
|
||||
assert!(local.into_iter().all(file_local_look));
|
||||
// DNG carries its own colour model
|
||||
assert!(!file_local_look(RawFormat::Dng));
|
||||
@@ -1228,4 +1322,77 @@ mod tests {
|
||||
reference.data.truncate(8);
|
||||
assert!(fit_pairs(&sensor, &reference).is_none());
|
||||
}
|
||||
|
||||
/// A busy scene whose JPEG is the camera curve of the scene two proxy pixels to the right (a
|
||||
/// lens-corrected preview): the fitted curve follows the camera's and has no flat steps.
|
||||
#[test]
|
||||
fn tone_fit_has_no_steps_on_a_misregistered_jpeg() {
|
||||
let camera = |x: f64| 0.6 * x.powf(0.55) / (1.0 + 0.2 * x);
|
||||
let (w, h) = (96usize, 64usize);
|
||||
let mut seed = 0x2545_f491_u32;
|
||||
let mut scene = Vec::with_capacity(w * h);
|
||||
for _ in 0..w * h {
|
||||
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
scene.push(0.01 * (5.0 * f64::from(seed >> 8) / f64::from(1u32 << 24)).exp());
|
||||
}
|
||||
let pairs: Vec<(f64, f64)> =
|
||||
(0..h).flat_map(|y| (0..w - 2).map(move |x| (y * w + x, y * w + x + 2))).map(|(i, j)| (scene[i], camera(scene[j]))).collect();
|
||||
let curve = fit_tone(pairs).unwrap();
|
||||
let mut sorted = scene.clone();
|
||||
sorted.sort_by(f64::total_cmp);
|
||||
let (lo, hi) = (sorted[sorted.len() / 50], sorted[sorted.len() * 49 / 50]);
|
||||
let mut x = lo;
|
||||
while x < hi {
|
||||
let (a, b) = (curve.apply(x as f32), curve.apply((x * 1.02) as f32));
|
||||
assert!(b > a, "flat step at {x}: {a} -> {b}");
|
||||
assert!((f64::from(a) / camera(x) - 1.0).abs() < 0.05, "at {x}: {a} vs {}", camera(x));
|
||||
x *= 1.02;
|
||||
}
|
||||
}
|
||||
|
||||
/// A synthetic photo whose JPEG is the camera look of the scene scaled per pixel by noise of
|
||||
/// `spread` (a pseudo-random, registration-free stand-in for local tone mapping): the per-bin
|
||||
/// median recovers the curve, while the JPEG's broadened distribution stretches the quantile fit.
|
||||
fn noisy_look(spread: f32) -> (Rgb32f, Rgb32f) {
|
||||
let known = Mat3([[1.8, -0.4, -0.1], [-0.2, 1.5, -0.1], [-0.05, -0.3, 1.7]]);
|
||||
let (w, h) = (96usize, 64usize);
|
||||
let mut sensor = Rgb32f::new(w, h);
|
||||
let mut reference = sensor.clone();
|
||||
let mut seed = 0x9e37_79b9_u32;
|
||||
let mut next = move || {
|
||||
seed = seed.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
|
||||
f32::from((seed >> 16) as u16) / 32768.0 - 1.0
|
||||
};
|
||||
for (src, dst) in sensor.data.iter_mut().zip(&mut reference.data) {
|
||||
let ev = 0.05 + (next() + 1.0) * 0.2;
|
||||
*src = [ev * (0.8 + (next() + 1.0) * 0.1), ev, ev * (0.8 + (next() + 1.0) * 0.1)];
|
||||
let p = known.apply_f32(*src);
|
||||
let y = luminance_2020(p);
|
||||
let scale = (1.0 - (-2.5 * y).exp()) / y * (1.0 + spread * next());
|
||||
*dst = p.map(|v| v * scale);
|
||||
}
|
||||
(sensor, reference)
|
||||
}
|
||||
|
||||
/// Quantile matching is tried first; the conditional-median fit is the fallback and is used
|
||||
/// only where the quantile look misses the acceptance gates, so no photo loses its look.
|
||||
#[test]
|
||||
fn tone_fit_order_quantile_first_conditional_median_as_fallback() {
|
||||
// Clean data: both fits pass, the default order uses the quantile curve (increasing).
|
||||
let (sensor, reference) = noisy_look(0.0);
|
||||
let default = fit_pairs(&sensor, &reference).expect("clean look");
|
||||
let quantile = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::Quantile]).expect("quantile passes");
|
||||
assert_eq!(default.tone, quantile.tone);
|
||||
// Noisy data: the quantile fit misses the gates, the conditional median still passes.
|
||||
let (sensor, reference) = noisy_look(0.33);
|
||||
assert!(fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::Quantile]).is_none(), "quantile alone misses the gates");
|
||||
let conditional = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::ConditionalMedian]).expect("conditional median passes");
|
||||
let default = fit_pairs(&sensor, &reference).expect("the fallback keeps the look");
|
||||
assert_eq!(default.tone, conditional.tone);
|
||||
// The order matters only through the first pass; with the fits swapped the conditional one wins even on clean data.
|
||||
let (sensor, reference) = noisy_look(0.0);
|
||||
let swapped = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::ConditionalMedian, ToneFit::Quantile]).unwrap();
|
||||
let conditional = fit_pairs_ordered(&sensor, &reference, None, &[ToneFit::ConditionalMedian]).unwrap();
|
||||
assert_eq!(swapped.tone, conditional.tone);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,9 +32,9 @@ use crate::media::ProbeInfo;
|
||||
|
||||
/// File extensions LightCraft imports (lower case).
|
||||
pub const EXTENSIONS: &[&str] = &[
|
||||
"jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "nrw", "arw", "raf", "orf", "rw2", "rwl", "raw", "pef", "psd", "jxl",
|
||||
"gif", "bmp", "heic", "avif", // containers LightCraft cannot decode but imports as preview only (their embedded JPEG)
|
||||
"iiq", "crw", "mrw", "x3f", "kdc", "mos", "erf",
|
||||
"jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "nrw", "arw", "raf", "orf", "rw2", "rwl", "raw", "pef", "srw", "psd",
|
||||
"jxl", "gif", "bmp", "heic", "avif", // containers LightCraft cannot decode but imports as preview only (their embedded JPEG)
|
||||
"iiq", "crw", "mrw", "x3f", "kdc", "mos", "erf", "3fr", "fff",
|
||||
];
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
|
||||
@@ -62,7 +62,7 @@ impl SettingsHashes {
|
||||
}
|
||||
|
||||
/// Bump when the pipeline's output changes, to invalidate cached thumbnails.
|
||||
pub const RENDER_CACHE_VERSION: u64 = 21;
|
||||
pub const RENDER_CACHE_VERSION: u64 = 22;
|
||||
|
||||
/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).
|
||||
pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];
|
||||
|
||||
@@ -12,9 +12,9 @@ use crate::backend::Backend;
|
||||
|
||||
/// File extensions recognised as photos when expanding folders.
|
||||
pub const PHOTO_EXTENSIONS: &[&str] = &[
|
||||
"jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "arw", "raf", "orf", "rw2", "rwl", "raw", "pef", "psd", "jxl", "gif",
|
||||
"bmp", "avif", // containers LightCraft cannot decode but imports as preview only (their embedded JPEG)
|
||||
"iiq", "crw", "mrw", "x3f", "kdc", "mos", "erf",
|
||||
"jpg", "jpeg", "png", "tif", "tiff", "webp", "dng", "cr2", "cr3", "nef", "arw", "raf", "orf", "rw2", "rwl", "raw", "pef", "srw", "psd", "jxl",
|
||||
"gif", "bmp", "avif", // containers LightCraft cannot decode but imports as preview only (their embedded JPEG)
|
||||
"iiq", "crw", "mrw", "x3f", "kdc", "mos", "erf", "3fr", "fff",
|
||||
];
|
||||
|
||||
/// Headless backend: a [`Session`] with filesystem hooks.
|
||||
|
||||
+41
-2
@@ -89,6 +89,39 @@ fn gain_table_map(ifd0: &Ifd, raw: &Ifd, order: ByteOrder) -> Option<GainTableMa
|
||||
}
|
||||
|
||||
pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
decode_as(bytes, mode, RawFormat::Dng)
|
||||
}
|
||||
|
||||
/// Whether a TIFF that is not a DNG describes a raw image the way DNG does: a full-resolution CFA IFD
|
||||
/// and DNG's own statement of the white balance (`AsShotNeutral`, in IFD0 or the raw IFD), in a coding
|
||||
/// [`decode_as`] reads: uncompressed, or lossless JPEG whose scan header the lossless decoder accepts.
|
||||
/// Such files (Hasselblad 3FR and FFF) carry no DNG version tag, so they are not DNGs, but nothing else
|
||||
/// in them needs a maker-specific reader. Without the white-balance tag nothing in the file says how
|
||||
/// to read its colour, and the file stays an undecodable raw.
|
||||
pub(crate) fn is_plain_cfa_tiff(tiff: &Tiff, bytes: &[u8]) -> bool {
|
||||
let Some(raw) = raw_ifd(tiff) else { return false };
|
||||
if !raw.contains(t::AS_SHOT_NEUTRAL) && !tiff.ifds.first().is_some_and(|i| i.contains(t::AS_SHOT_NEUTRAL)) {
|
||||
return false;
|
||||
}
|
||||
let Ok(info) = raw.image() else { return false };
|
||||
if info.samples_per_pixel != 1 || !(1..=16).contains(&info.bits()) || info.sample_format == 3 {
|
||||
return false;
|
||||
}
|
||||
match info.compression {
|
||||
t::compression::NONE => true,
|
||||
t::compression::JPEG => info
|
||||
.chunks(bytes.len() as u64)
|
||||
.first()
|
||||
.and_then(|c| lightcraft_tiff::image::chunk_bytes(bytes, c))
|
||||
.is_some_and(|src| crate::ljpeg::frame_info(src).is_ok()),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// [`decode`] for the TIFF-based raws that follow the DNG layout without being DNGs (`format` says
|
||||
/// which). They differ in one respect: when the raw IFD states no `CFAPattern`, the layout is RGGB
|
||||
/// anchored at the sensor origin.
|
||||
pub(crate) fn decode_as(bytes: &[u8], mode: Mode, format: RawFormat) -> Result<RawImage> {
|
||||
let tiff = Tiff::parse(bytes)?;
|
||||
let ifd0 = &tiff.ifds[0];
|
||||
let raw = raw_ifd(&tiff).ok_or_else(|| RawError::Corrupt("DNG without a raw image IFD".into()))?;
|
||||
@@ -152,7 +185,13 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
[r, c] if (1..=16).contains(r) && (1..=16).contains(c) => (*r as usize, *c as usize),
|
||||
_ => return Err(RawError::Corrupt("bad CFARepeatPatternDim".into())),
|
||||
};
|
||||
let pat = raw.bytes(t::CFA_PATTERN_EP).ok_or_else(|| RawError::Corrupt("missing CFAPattern".into()))?;
|
||||
// a TIFF that is not a DNG may leave the layout out: RGGB from the sensor origin
|
||||
let assumed = [0u8, 1, 1, 2];
|
||||
let pat = match raw.bytes(t::CFA_PATTERN_EP) {
|
||||
Some(p) => p,
|
||||
None if format != RawFormat::Dng && (rows, cols) == (2, 2) => &assumed,
|
||||
None => return Err(RawError::Corrupt("missing CFAPattern".into())),
|
||||
};
|
||||
if pat.len() != rows * cols {
|
||||
return Err(RawError::Corrupt("CFAPattern size mismatch".into()));
|
||||
}
|
||||
@@ -178,7 +217,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
metadata.width = Some(crop.width as u32);
|
||||
metadata.height = Some(crop.height as u32);
|
||||
let img = RawImage {
|
||||
format: RawFormat::Dng,
|
||||
format,
|
||||
width: w,
|
||||
height: h,
|
||||
cpp,
|
||||
|
||||
+211
-12
@@ -9,17 +9,24 @@
|
||||
//! tiles decode bit-exactly, whatever bit depth the encoder declared relative to the IFD's
|
||||
//! `BitsPerSample`. Float samples are returned as decoded.
|
||||
//!
|
||||
//! We never request an output colour encoding, so jxl-oxide renders in the codestream's *original*
|
||||
//! colour encoding: for lossless (non-XYB) data that is the identity, keeping the values in the raw IFD's
|
||||
//! sample space. Lossy (XYB) tiles are converted from XYB back to that signalled encoding by jxl-oxide;
|
||||
//! [INFERENCE] DNG writers signal a linear encoding matching the raw data there, so the result is the raw
|
||||
//! sample space up to the codec's loss — not verified against a real lossy JXL DNG.
|
||||
//! Lossless (non-XYB) tiles are rendered in the codestream's own colour encoding, which is the identity,
|
||||
//! keeping the values in the raw IFD's sample space. Lossy (XYB) tiles are decoded by jxl-oxide to linear
|
||||
//! sRGB primaries; the codestream's declared primaries are then restored with a plain 3×3 matrix. We
|
||||
//! never let jxl-oxide convert to the declared encoding itself: that step gamut-maps and clips to
|
||||
//! [0, 1], which destroys raw values above 1.0 (a `WhiteLevel` of 32768 puts them far above) and
|
||||
//! desaturates. The result is the linear sample space the DNG's `WhiteLevel` refers to, up to the
|
||||
//! codec's loss, and can exceed 1.0 or dip below 0.
|
||||
//!
|
||||
//! Every tile is decoded with wide (32-bit) modular buffers. A lossy tile whose header sets
|
||||
//! `modular_16bit_buffers` can still carry values that need more than 16 bits; the narrow buffers then
|
||||
//! fail with `InvalidAnsStream` or `UnexpectedEof`.
|
||||
|
||||
use crate::tiffraw::ChunkPx;
|
||||
use crate::{RawError, Result};
|
||||
use jxl_oxide::color::{ColourEncoding, ColourSpace, Primaries, TransferFunction, WhitePoint};
|
||||
use jxl_oxide::frame::Encoding;
|
||||
use jxl_oxide::image::BitDepth;
|
||||
use jxl_oxide::{AllocTracker, JxlImage, JxlThreadPool};
|
||||
use jxl_oxide::{AllocTracker, EnumColourEncoding, JxlImage, JxlThreadPool, RenderingIntent};
|
||||
|
||||
/// Decoder memory (bytes) allowed per sample of a tile. jxl-oxide 0.12 peaks at about three 32-bit
|
||||
/// buffers per sample: measured ≤ 10.3 bytes for lossless (modular) tiles and ≤ 13.5 bytes per
|
||||
@@ -29,6 +36,99 @@ const BYTES_PER_SAMPLE: usize = 16;
|
||||
/// on a 16 × 16 tile.
|
||||
const MARGIN_BYTES: usize = 64 << 10;
|
||||
|
||||
/// Linear sRGB primaries with a D65 white point (the space jxl-oxide decodes XYB to) and the matrix that
|
||||
/// takes a linear colour in it to the codestream's declared primaries (row-major, D65 white point).
|
||||
type Matrix = [[f64; 3]; 3];
|
||||
|
||||
const D65: [f64; 2] = [0.3127, 0.3290];
|
||||
const SRGB: [[f64; 2]; 3] = [[0.640, 0.330], [0.300, 0.600], [0.150, 0.060]];
|
||||
const BT2100: [[f64; 2]; 3] = [[0.708, 0.292], [0.170, 0.797], [0.131, 0.046]];
|
||||
const P3: [[f64; 2]; 3] = [[0.680, 0.320], [0.265, 0.690], [0.150, 0.060]];
|
||||
|
||||
fn inverse(m: Matrix) -> Matrix {
|
||||
let c =
|
||||
|i: usize, j: usize| m[(i + 1) % 3][(j + 1) % 3] * m[(i + 2) % 3][(j + 2) % 3] - m[(i + 1) % 3][(j + 2) % 3] * m[(i + 2) % 3][(j + 1) % 3];
|
||||
let det = m[0][0] * c(0, 0) + m[0][1] * c(0, 1) + m[0][2] * c(0, 2);
|
||||
let mut out = [[0.0; 3]; 3];
|
||||
for (i, row) in out.iter_mut().enumerate() {
|
||||
for (j, v) in row.iter_mut().enumerate() {
|
||||
*v = c(j, i) / det;
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn mul(a: Matrix, b: Matrix) -> Matrix {
|
||||
let mut out = [[0.0; 3]; 3];
|
||||
for (i, row) in out.iter_mut().enumerate() {
|
||||
for (j, v) in row.iter_mut().enumerate() {
|
||||
*v = (0..3).map(|k| a[i][k] * b[k][j]).sum();
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// Linear RGB → XYZ for primaries `p` (xy) and the D65 white point.
|
||||
fn rgb_to_xyz(p: [[f64; 2]; 3]) -> Matrix {
|
||||
let xyz = |[x, y]: [f64; 2]| [x / y, 1.0, (1.0 - x - y) / y];
|
||||
let cols = [xyz(p[0]), xyz(p[1]), xyz(p[2])];
|
||||
let prim = [[cols[0][0], cols[1][0], cols[2][0]], [1.0, 1.0, 1.0], [cols[0][2], cols[1][2], cols[2][2]]];
|
||||
let w = xyz(D65);
|
||||
let inv = inverse(prim);
|
||||
let s: [f64; 3] = std::array::from_fn(|i| (0..3).map(|k| inv[i][k] * w[k]).sum());
|
||||
std::array::from_fn(|i| std::array::from_fn(|j| prim[i][j] * s[j]))
|
||||
}
|
||||
|
||||
/// How a chunk is rendered by jxl-oxide.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
enum Output {
|
||||
/// In the codestream's own colour encoding (non-XYB: the identity).
|
||||
AsCoded,
|
||||
/// XYB (lossy) data: linear sRGB primaries from the decoder, then `Some(matrix)` to the declared
|
||||
/// primaries (`None` when those are sRGB's).
|
||||
FromXyb(Option<[[f32; 3]; 3]>),
|
||||
}
|
||||
|
||||
/// Decide how to render a chunk from its header: whether it is XYB-coded, the declared colour
|
||||
/// encoding and the intensity target. Lossy data is only accepted when the declared encoding is a
|
||||
/// linear RGB one with a D65 white point (what a raw DNG writer signals) and no HDR tone mapping would
|
||||
/// apply, otherwise the samples cannot be restored faithfully.
|
||||
fn output_for(xyb: bool, encoding: &ColourEncoding, intensity_target: f32, planes: usize) -> Result<Output> {
|
||||
if !xyb {
|
||||
return Ok(Output::AsCoded);
|
||||
}
|
||||
let unsupported = |what: &str| RawError::Unsupported(format!("lossy JPEG XL tile with {what}"));
|
||||
if planes != 3 {
|
||||
return Err(unsupported("a single colour plane"));
|
||||
}
|
||||
let ColourEncoding::Enum(e) = encoding else {
|
||||
return Err(unsupported("an ICC profile instead of a colour encoding"));
|
||||
};
|
||||
if e.colour_space != ColourSpace::Rgb || e.tf != TransferFunction::Linear {
|
||||
return Err(unsupported("a declared encoding that is not linear RGB"));
|
||||
}
|
||||
if e.white_point != WhitePoint::D65 {
|
||||
return Err(unsupported("a white point other than D65"));
|
||||
}
|
||||
if intensity_target > 255.0 {
|
||||
return Err(unsupported("an intensity target above 255 nits"));
|
||||
}
|
||||
let primaries = match e.primaries {
|
||||
Primaries::Srgb => return Ok(Output::FromXyb(None)),
|
||||
Primaries::Bt2100 => BT2100,
|
||||
Primaries::P3 => P3,
|
||||
Primaries::Custom { red, green, blue } => [red, green, blue].map(|c| c.as_float().map(f64::from)),
|
||||
};
|
||||
if primaries.iter().any(|p| p[1] <= 0.0) {
|
||||
return Err(unsupported("invalid primaries"));
|
||||
}
|
||||
let m = mul(inverse(rgb_to_xyz(primaries)), rgb_to_xyz(SRGB));
|
||||
if m.iter().flatten().any(|v| !v.is_finite()) {
|
||||
return Err(unsupported("degenerate primaries"));
|
||||
}
|
||||
Ok(Output::FromXyb(Some(m.map(|r| r.map(|v| v as f32)))))
|
||||
}
|
||||
|
||||
fn corrupt(e: impl std::fmt::Display) -> RawError {
|
||||
RawError::Corrupt(format!("JPEG XL tile: {e}"))
|
||||
}
|
||||
@@ -37,7 +137,7 @@ fn corrupt(e: impl std::fmt::Display) -> RawError {
|
||||
/// nominal `cw × ch`, or the part of it inside the image `vw × vh` (edge tiles), `cpp` colour planes,
|
||||
/// a sample type matching the IFD. Returns the decoder and the codestream's integer maximum (0 for float).
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn open(src: &[u8], cw: usize, ch: usize, vw: usize, vh: usize, cpp: usize, bits: u32, float: bool) -> Result<(JxlImage, f32)> {
|
||||
fn open(src: &[u8], cw: usize, ch: usize, vw: usize, vh: usize, cpp: usize, bits: u32, float: bool) -> Result<(JxlImage, f32, Output)> {
|
||||
if !matches!(cpp, 1 | 3) {
|
||||
return Err(RawError::Unsupported(format!("JPEG XL with {cpp} samples per pixel")));
|
||||
}
|
||||
@@ -49,8 +149,9 @@ fn open(src: &[u8], cw: usize, ch: usize, vw: usize, vh: usize, cpp: usize, bits
|
||||
let pixels = cw.saturating_mul(ch);
|
||||
let budget = |planes: usize| pixels.saturating_mul(planes).saturating_mul(BYTES_PER_SAMPLE);
|
||||
let tracker = AllocTracker::with_limit(budget(cpp).saturating_add(MARGIN_BYTES));
|
||||
let img = JxlImage::builder()
|
||||
let mut img = JxlImage::builder()
|
||||
.pool(JxlThreadPool::none()) // chunks are already decoded in parallel
|
||||
.force_wide_buffers(true) // see the module documentation
|
||||
.alloc_tracker(tracker.clone())
|
||||
.read(std::io::Cursor::new(src))
|
||||
.map_err(corrupt)?;
|
||||
@@ -75,7 +176,11 @@ fn open(src: &[u8], cw: usize, ch: usize, vw: usize, vh: usize, cpp: usize, bits
|
||||
BitDepth::FloatSample { .. } if float => 0.0,
|
||||
d => return Err(RawError::Corrupt(format!("JPEG XL tile sample type {d:?} does not match the IFD"))),
|
||||
};
|
||||
Ok((img, max))
|
||||
let output = output_for(header.metadata.xyb_encoded, &header.metadata.colour_encoding, header.metadata.tone_mapping.intensity_target, planes)?;
|
||||
if matches!(output, Output::FromXyb(_)) {
|
||||
img.request_color_encoding(EnumColourEncoding::srgb_linear(RenderingIntent::Relative));
|
||||
}
|
||||
Ok((img, max, output))
|
||||
}
|
||||
|
||||
/// Check the first chunk's headers ([`open`]) without decoding pixels (header-only probing).
|
||||
@@ -88,7 +193,7 @@ pub(crate) fn check(src: &[u8], cw: usize, ch: usize, vw: usize, vh: usize, cpp:
|
||||
/// in-image size `vw × vh` fills the top-left of that, the rest stays 0.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) fn decode(src: &[u8], cw: usize, ch: usize, vw: usize, vh: usize, cpp: usize, bits: u32, float: bool) -> Result<ChunkPx> {
|
||||
let (img, max) = open(src, cw, ch, vw, vh, cpp, bits, float)?;
|
||||
let (img, max, output) = open(src, cw, ch, vw, vh, cpp, bits, float)?;
|
||||
if img.num_loaded_keyframes() == 0 {
|
||||
return Err(corrupt("no complete frame"));
|
||||
}
|
||||
@@ -100,22 +205,116 @@ pub(crate) fn decode(src: &[u8], cw: usize, ch: usize, vw: usize, vh: usize, cpp
|
||||
}
|
||||
let n = cw.saturating_mul(ch).saturating_mul(cpp);
|
||||
let rows = fb.buf().chunks_exact(jw * fc);
|
||||
// lossy data: restore the declared primaries (see the module documentation)
|
||||
let matrix = match output {
|
||||
Output::FromXyb(m) => m,
|
||||
Output::AsCoded => None,
|
||||
};
|
||||
let restore = |s: &[f32], d: &mut [f32]| match matrix {
|
||||
Some(m) => {
|
||||
for (d, row) in d.iter_mut().zip(&m) {
|
||||
*d = row[0] * s[0] + row[1] * s[1] + row[2] * s[2];
|
||||
}
|
||||
}
|
||||
None => d.copy_from_slice(&s[..cpp]),
|
||||
};
|
||||
Ok(if float {
|
||||
let mut out = vec![0f32; n];
|
||||
for (dst, src) in out.chunks_exact_mut(cw * cpp).zip(rows) {
|
||||
for (d, s) in dst.chunks_exact_mut(cpp).zip(src.chunks_exact(fc)) {
|
||||
d.iter_mut().zip(s).for_each(|(d, s)| *d = *s);
|
||||
restore(s, d);
|
||||
}
|
||||
}
|
||||
ChunkPx::F32(out)
|
||||
} else {
|
||||
let mut out = vec![0u16; n];
|
||||
let mut px = [0f32; 3];
|
||||
for (dst, src) in out.chunks_exact_mut(cw * cpp).zip(rows) {
|
||||
for (d, s) in dst.chunks_exact_mut(cpp).zip(src.chunks_exact(fc)) {
|
||||
restore(s, &mut px[..cpp]);
|
||||
// `as` saturates (NaN → 0): lossy overshoot clips to the integer range
|
||||
d.iter_mut().zip(s).for_each(|(d, s)| *d = (s * max).round() as u16);
|
||||
d.iter_mut().zip(&px).for_each(|(d, s)| *d = (s * max).round() as u16);
|
||||
}
|
||||
}
|
||||
ChunkPx::U16(out)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use jxl_oxide::color::Customxy;
|
||||
|
||||
fn linear(primaries: Primaries) -> ColourEncoding {
|
||||
ColourEncoding::Enum(EnumColourEncoding {
|
||||
colour_space: ColourSpace::Rgb,
|
||||
white_point: WhitePoint::D65,
|
||||
primaries,
|
||||
tf: TransferFunction::Linear,
|
||||
rendering_intent: RenderingIntent::Perceptual,
|
||||
})
|
||||
}
|
||||
|
||||
fn matrix(o: Output) -> [[f32; 3]; 3] {
|
||||
match o {
|
||||
Output::FromXyb(Some(m)) => m,
|
||||
other => panic!("expected a matrix, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_that_is_not_xyb_is_rendered_as_coded() {
|
||||
// lossless tiles: whatever the header declares, the decoder must not convert anything
|
||||
for e in
|
||||
[linear(Primaries::Srgb), linear(Primaries::Bt2100), ColourEncoding::Enum(EnumColourEncoding::srgb_gamma22(RenderingIntent::Relative))]
|
||||
{
|
||||
assert_eq!(output_for(false, &e, 255.0, 3).unwrap(), Output::AsCoded);
|
||||
assert_eq!(output_for(false, &e, 10_000.0, 1).unwrap(), Output::AsCoded);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lossy_data_with_srgb_primaries_needs_no_matrix() {
|
||||
assert_eq!(output_for(true, &linear(Primaries::Srgb), 255.0, 3).unwrap(), Output::FromXyb(None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lossy_data_is_restored_to_the_declared_primaries() {
|
||||
// linear sRGB to linear BT.2020/BT.2100 primaries (D65), as published in ITU-R BT.2087
|
||||
let expect = [[0.6274, 0.3293, 0.0433], [0.0691, 0.9195, 0.0114], [0.0164, 0.0880, 0.8956]];
|
||||
let m = matrix(output_for(true, &linear(Primaries::Bt2100), 255.0, 3).unwrap());
|
||||
for (r, e) in m.iter().zip(&expect) {
|
||||
for (v, e) in r.iter().zip(e) {
|
||||
assert!((v - e).abs() < 1e-3, "{m:?}");
|
||||
}
|
||||
// the white point maps to itself: rows sum to one
|
||||
assert!((r.iter().sum::<f32>() - 1.0).abs() < 1e-5, "{m:?}");
|
||||
}
|
||||
// custom primaries equal to the named ones give the same matrix
|
||||
let xy = |[x, y]: [f64; 2]| Customxy { x: (x * 1e6) as i32, y: (y * 1e6) as i32 };
|
||||
let custom = Primaries::Custom { red: xy(BT2100[0]), green: xy(BT2100[1]), blue: xy(BT2100[2]) };
|
||||
let c = matrix(output_for(true, &linear(custom), 255.0, 3).unwrap());
|
||||
assert!(c.iter().flatten().zip(m.iter().flatten()).all(|(a, b)| (a - b).abs() < 1e-5), "{c:?} {m:?}");
|
||||
// the sRGB primaries lie inside the wider P3 gamut: the full-saturation sRGB green is a valid P3 colour
|
||||
let inv = matrix(output_for(true, &linear(Primaries::P3), 255.0, 3).unwrap());
|
||||
let srgb_green_in_p3 = [inv[0][1], inv[1][1], inv[2][1]];
|
||||
assert!(srgb_green_in_p3.iter().all(|v| (0.0..=1.0).contains(v)), "sRGB green lies inside P3");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lossy_data_that_cannot_be_restored_is_unsupported() {
|
||||
let bad = |xyb_planes: usize, e: ColourEncoding, it: f32| matches!(output_for(true, &e, it, xyb_planes), Err(RawError::Unsupported(_)));
|
||||
assert!(bad(1, linear(Primaries::Srgb), 255.0), "grey");
|
||||
assert!(bad(3, linear(Primaries::Srgb), 10_000.0), "HDR intensity target would tone-map");
|
||||
assert!(bad(3, ColourEncoding::Enum(EnumColourEncoding::srgb_gamma22(RenderingIntent::Relative)), 255.0), "non-linear");
|
||||
let mut e = EnumColourEncoding::srgb_linear(RenderingIntent::Relative);
|
||||
e.white_point = WhitePoint::Dci;
|
||||
assert!(bad(3, ColourEncoding::Enum(e.clone()), 255.0), "white point");
|
||||
e.white_point = WhitePoint::D65;
|
||||
e.primaries =
|
||||
Primaries::Custom { red: Customxy { x: 0, y: 0 }, green: Customxy { x: 300_000, y: 600_000 }, blue: Customxy { x: 150_000, y: 60_000 } };
|
||||
assert!(bad(3, ColourEncoding::Enum(e.clone()), 255.0), "degenerate primaries");
|
||||
e.colour_space = ColourSpace::Grey;
|
||||
assert!(bad(3, ColourEncoding::Enum(e.clone()), 255.0), "grey declared");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,6 +96,10 @@ pub enum RawFormat {
|
||||
Mrw,
|
||||
/// Sigma / Foveon X3F (`FOVb`).
|
||||
X3f,
|
||||
/// A TIFF-based raw that has no DNG version tag but describes itself the way DNG does (a
|
||||
/// full-resolution CFA image with black/white levels, a default crop and `AsShotNeutral`):
|
||||
/// Hasselblad 3FR and FFF. Read by the DNG reader.
|
||||
CfaTiff,
|
||||
/// Another TIFF-based raw (3FR, IIQ, ERF, KDC, DCR, MOS, …).
|
||||
OtherTiff,
|
||||
}
|
||||
@@ -114,6 +118,8 @@ impl RawFormat {
|
||||
| RawFormat::Raf
|
||||
| RawFormat::Rw2
|
||||
| RawFormat::Pef
|
||||
| RawFormat::CfaTiff
|
||||
| RawFormat::Srw
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -172,6 +178,9 @@ pub fn probe(bytes: &[u8]) -> Option<RawFormat> {
|
||||
if make.starts_with("SAMSUNG") {
|
||||
return Some(RawFormat::Srw);
|
||||
}
|
||||
if dng::is_plain_cfa_tiff(&t, bytes) {
|
||||
return Some(RawFormat::CfaTiff);
|
||||
}
|
||||
if has_cfa || thumbnail_shell(&t, bytes.len()).is_some() || is_preview_container(ifd0) {
|
||||
return Some(RawFormat::OtherTiff);
|
||||
}
|
||||
@@ -266,6 +275,15 @@ fn thumbnail_shell(t: &Tiff, len: usize) -> Option<ThumbnailShell> {
|
||||
/// Why [`decode`] gives up on a file [`probe`] called [`RawFormat::OtherTiff`].
|
||||
fn other_tiff_reason(bytes: &[u8]) -> String {
|
||||
let t = Tiff::parse_with(bytes, &lightcraft_tiff::ParseOptions { max_ifds: 256, ..Default::default() }).ok();
|
||||
// a DNG-style CFA IFD whose lossless JPEG the lossless decoder rejects: say why
|
||||
if let Some(t) = &t
|
||||
&& let Some(info) = dng::raw_ifd(t).and_then(|i| i.image().ok())
|
||||
&& info.compression == lightcraft_tiff::tags::compression::JPEG
|
||||
&& let Some(src) = info.chunks(bytes.len() as u64).first().and_then(|c| lightcraft_tiff::image::chunk_bytes(bytes, c))
|
||||
&& let Err(e) = ljpeg::frame_info(src)
|
||||
{
|
||||
return format!("raw image coded as lossless JPEG that is not decoded yet ({e})");
|
||||
}
|
||||
match t.as_ref().filter(|t| !has_raw_ifd(t)).and_then(|t| thumbnail_shell(t, bytes.len())) {
|
||||
Some(s) => format!(
|
||||
"{}x{} raw image in a private block, not decoded yet (the file's first image is a {}x{} reduced copy)",
|
||||
@@ -315,6 +333,8 @@ fn decode_with(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
RawFormat::Rw2 => vendor::rw2::decode(bytes, mode),
|
||||
RawFormat::Pef => vendor::pef::decode(bytes, mode),
|
||||
RawFormat::Orf => vendor::orf::decode(bytes, mode),
|
||||
RawFormat::CfaTiff => dng::decode_as(bytes, mode, RawFormat::CfaTiff),
|
||||
RawFormat::Srw => vendor::srw::decode(bytes, mode),
|
||||
RawFormat::OtherTiff => Err(RawError::Unsupported(other_tiff_reason(bytes))),
|
||||
other => Err(RawError::Unsupported(format!("{other:?} files are not decoded yet"))),
|
||||
}
|
||||
@@ -929,6 +949,125 @@ mod tests {
|
||||
assert_eq!(probe(&write(&[ifd])), Some(RawFormat::OtherTiff));
|
||||
}
|
||||
|
||||
// --- TIFF raws that describe their raw IFD the way DNG does but carry no DNG version tag ---
|
||||
|
||||
/// 8 x 6 samples, 16 bits, value `1000 + 100 y + x`.
|
||||
fn cfa_samples() -> Vec<u16> {
|
||||
(0..48u16).map(|i| 1000 + 100 * (i / 8) + i % 8).collect()
|
||||
}
|
||||
|
||||
/// A small raw in the layout of the Hasselblad 3FR/FFF family: IFD0 is a reduced RGB image with the
|
||||
/// colour tags, a SubIFD holds the full-resolution CFA image (no `CFAPattern`, DNG-style levels and
|
||||
/// crop). `strip` is the CFA image's data and `compression` its coding.
|
||||
fn plain_cfa_tiff(compression: u16, strip: Vec<u8>, pattern: Option<[u8; 4]>) -> Vec<u8> {
|
||||
let mut raw = IfdBuilder::new();
|
||||
raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0]));
|
||||
raw.set(t::IMAGE_WIDTH, Value::Long(vec![8]));
|
||||
raw.set(t::IMAGE_LENGTH, Value::Long(vec![6]));
|
||||
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16]));
|
||||
raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1]));
|
||||
raw.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::CFA]));
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![compression]));
|
||||
raw.set(t::BLACK_LEVEL, Value::Rational(vec![(256, 1)]));
|
||||
raw.set(t::WHITE_LEVEL, Value::Long(vec![60000]));
|
||||
raw.set(t::DEFAULT_CROP_ORIGIN, Value::Short(vec![2, 2]));
|
||||
raw.set(t::DEFAULT_CROP_SIZE, Value::Short(vec![4, 2]));
|
||||
if let Some(p) = pattern {
|
||||
raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2]));
|
||||
raw.set(t::CFA_PATTERN_EP, Value::Byte(p.to_vec()));
|
||||
}
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: 6, strips: vec![strip] });
|
||||
let mut ifd0 = rgb_ifd(4, 3);
|
||||
ifd0.set(t::MAKE, Value::Ascii("Hasselblad".into()));
|
||||
ifd0.set(t::COLOR_MATRIX_1, Value::SRational(vec![(5, 10), (-1, 10), (0, 10), (-5, 10), (12, 10), (3, 10), (-1, 10), (2, 10), (6, 10)]));
|
||||
ifd0.set(t::AS_SHOT_NEUTRAL, Value::Rational(vec![(2, 5), (1, 1), (3, 5)]));
|
||||
ifd0.add_sub_ifd(raw);
|
||||
write(&[ifd0])
|
||||
}
|
||||
|
||||
fn le_words(v: &[u16]) -> Vec<u8> {
|
||||
v.iter().flat_map(|x| x.to_le_bytes()).collect()
|
||||
}
|
||||
|
||||
/// Known answer: samples, black and white level, crop and colour come from the file's own tags; the
|
||||
/// layout it does not state is RGGB.
|
||||
#[test]
|
||||
fn uncompressed_cfa_tiff_without_a_dng_version_decodes() {
|
||||
let bytes = plain_cfa_tiff(1, le_words(&cfa_samples()), None);
|
||||
assert_eq!(probe(&bytes), Some(RawFormat::CfaTiff));
|
||||
let r = decode(&bytes).unwrap();
|
||||
assert_eq!((r.format, r.width, r.height, r.bits), (RawFormat::CfaTiff, 8, 6, 16));
|
||||
assert_eq!(r.data, RawData::U16(cfa_samples()));
|
||||
assert_eq!(r.cfa.as_ref().unwrap().name(), "RGGB");
|
||||
assert_eq!((r.black.mean(), r.white.clone()), (256.0, vec![60000.0]));
|
||||
assert_eq!(r.crop, Rect::new(2, 2, 4, 2));
|
||||
assert!(color::has_matrix(&r.color));
|
||||
assert_eq!(r.color.as_shot_neutral, Some([0.4, 1.0, 0.6]));
|
||||
// headers only: the same description without samples
|
||||
let i = probe_info(&bytes).unwrap();
|
||||
assert_eq!((i.format, i.cfa.as_ref().map(Cfa::name), i.crop), (RawFormat::CfaTiff, Some("RGGB".to_string()), Rect::new(2, 2, 4, 2)));
|
||||
}
|
||||
|
||||
/// Without DNG's white-balance tag the file is left alone (a Kodak or Sinar TIFF with a CFA IFD, say).
|
||||
#[test]
|
||||
fn cfa_tiff_without_a_white_balance_tag_is_not_claimed() {
|
||||
let mut raw = IfdBuilder::new();
|
||||
raw.set(t::IMAGE_WIDTH, Value::Long(vec![8]));
|
||||
raw.set(t::IMAGE_LENGTH, Value::Long(vec![6]));
|
||||
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![16]));
|
||||
raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1]));
|
||||
raw.set(t::PHOTOMETRIC, Value::Short(vec![t::photometric::CFA]));
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![1]));
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: 6, strips: vec![le_words(&cfa_samples())] });
|
||||
let mut ifd0 = rgb_ifd(4, 3);
|
||||
ifd0.add_sub_ifd(raw);
|
||||
assert_eq!(probe(&write(&[ifd0])), Some(RawFormat::OtherTiff));
|
||||
}
|
||||
|
||||
/// A stated `CFAPattern` wins over the RGGB default.
|
||||
#[test]
|
||||
fn cfa_tiff_keeps_a_stated_pattern() {
|
||||
let bytes = plain_cfa_tiff(1, le_words(&cfa_samples()), Some([2, 1, 1, 0]));
|
||||
assert_eq!(decode(&bytes).unwrap().cfa.unwrap().name(), "BGGR");
|
||||
}
|
||||
|
||||
/// The lossless-JPEG coding of the same layout goes through the lossless decoder (one strip for the
|
||||
/// whole image, as the 3FR/FFF files have it).
|
||||
#[test]
|
||||
fn lossless_jpeg_cfa_tiff_decodes() {
|
||||
let strip = ljpeg::encode(&cfa_samples(), 8, 6, 1, 16, 1, 0);
|
||||
let bytes = plain_cfa_tiff(7, strip, None);
|
||||
assert_eq!(probe(&bytes), Some(RawFormat::CfaTiff));
|
||||
let r = decode(&bytes).unwrap();
|
||||
assert_eq!(r.data, RawData::U16(cfa_samples()));
|
||||
assert_eq!(r.cfa.unwrap().name(), "RGGB");
|
||||
}
|
||||
|
||||
/// A lossless-JPEG scan whose predictor selection value is outside 0 to 7 (the 3FR/FFF files use 8) is
|
||||
/// not a coding the lossless decoder knows: the file stays an undecodable raw, with that as the reason,
|
||||
/// and keeps the stand-in preview an unsupported container gets.
|
||||
#[test]
|
||||
fn cfa_tiff_with_an_unknown_predictor_stays_undecodable() {
|
||||
let mut strip = ljpeg::encode(&cfa_samples(), 8, 6, 1, 16, 1, 0);
|
||||
let sos = strip.windows(2).position(|w| w == [0xff, 0xda]).unwrap();
|
||||
let ss = sos + 2 + 2 + 1 + 2;
|
||||
assert_eq!(strip[ss], 1, "selection value byte");
|
||||
strip[ss] = 8;
|
||||
assert!(matches!(ljpeg::decode(&strip, 1 << 20), Err(RawError::Unsupported(w)) if w.contains("selection value 8")));
|
||||
let bytes = plain_cfa_tiff(7, strip, None);
|
||||
assert_eq!(probe(&bytes), Some(RawFormat::OtherTiff));
|
||||
assert!(!RawFormat::OtherTiff.is_supported());
|
||||
let Err(RawError::Unsupported(why)) = decode(&bytes) else { panic!("expected Unsupported") };
|
||||
assert!(why.contains("lossless JPEG") && why.contains("selection value 8"), "{why}");
|
||||
}
|
||||
|
||||
/// Other codings of a CFA IFD (here Deflate) are not claimed by the plain reader.
|
||||
#[test]
|
||||
fn cfa_tiff_with_another_coding_is_not_claimed() {
|
||||
let bytes = plain_cfa_tiff(8, vec![0; 96], None);
|
||||
assert_eq!(probe(&bytes), Some(RawFormat::OtherTiff));
|
||||
}
|
||||
|
||||
/// A Samsung-branded body whose maker note has a Pentax layout is read by the Pentax reader; another
|
||||
/// Samsung file stays with the Samsung format.
|
||||
#[test]
|
||||
|
||||
@@ -325,8 +325,11 @@ fn parse_header(d: &[u8]) -> Result<Header, RawError> {
|
||||
}
|
||||
h.predictor = seg[1 + 2 * ns];
|
||||
h.pt = seg[3 + 2 * ns] & 15;
|
||||
if !(0..=7).contains(&h.predictor) {
|
||||
return Err(err("bad predictor"));
|
||||
if h.predictor > 7 {
|
||||
return Err(RawError::Unsupported(format!(
|
||||
"lossless JPEG scan with predictor selection value {} (T.81 defines 0 to 7)",
|
||||
h.predictor
|
||||
)));
|
||||
}
|
||||
if h.pt >= h.precision {
|
||||
return Err(err("bad point transform"));
|
||||
|
||||
@@ -108,6 +108,10 @@ pub fn read_image(data: &[u8], info: &ImageInfo, order: ByteOrder, packing: Pack
|
||||
let (c, px) = match r {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
// a JPEG XL tile that cannot be decoded is not a partial image: the file stays preview-only
|
||||
if info.compression == comp::JPEG_XL {
|
||||
return Err(e);
|
||||
}
|
||||
first_err.get_or_insert(e);
|
||||
continue;
|
||||
}
|
||||
|
||||
Vendored
+1
@@ -12,6 +12,7 @@ pub mod pef;
|
||||
pub mod raf;
|
||||
mod rafc;
|
||||
pub mod rw2;
|
||||
pub mod srw;
|
||||
|
||||
use crate::{BlackLevel, Cfa, Rect};
|
||||
use lightcraft_tiff::Tiff;
|
||||
|
||||
Vendored
+541
@@ -0,0 +1,541 @@
|
||||
//! Samsung SRW: the variants that store the sensor samples uncompressed.
|
||||
//!
|
||||
//! Sources: TIFF 6.0 (container: a big- or little-endian TIFF whose IFD0 points at sub-IFDs, one of them the raw
|
||||
//! image), Exif 2.3 (`CFAPattern`, `PixelXDimension`, `PixelYDimension`; maker note `0x927c`), the ExifTool Samsung
|
||||
//! tag-name documentation (`SensorAreas` `0xa010`, `EncryptionKey` `0xa020`, `WB_RGGBLevelsUncorrected` `0xa021`,
|
||||
//! `WB_RGGBLevelsBlack` `0xa028`, `RawData` `0xa048`) and our own black-box analysis of CC0 samples from raw.pixls.us
|
||||
//! (NX5, NX10, NX11, NX20, NX200, NX210, NX1000, NX1100, EX1, WB2000; every number below was measured on them):
|
||||
//!
|
||||
//! - The raw image is the IFD whose `Compression` is a private value (32769..=32773); its `CFAPattern` (`0x828e`)
|
||||
//! names the 2x2 layout, anchored at the first sample of the stored array. Checked on the files above: the colours
|
||||
//! of the decoded mosaic follow the camera's JPEG best with exactly that layout (`corpus_samsung_srw`).
|
||||
//! - Compression 32769 (EX1, WB2000): one 16-bit little-endian word per sample (the TIFF is big-endian, the words are
|
||||
//! not); the strip holds exactly `width * height * 2` bytes.
|
||||
//! - Compression 32770 with exactly `width * height * 12 / 8` strip bytes (NX5, NX10, NX11, NX20, NX200, NX210, NX1000,
|
||||
//! NX1100): 12-bit samples, two in three bytes, rows without padding. The NX5/NX10/NX11 files read them MSB-first,
|
||||
//! the others LSB-first; the file doesn't say which, the image does: in the wrong order neighbouring same-colour
|
||||
//! samples differ 5 to 11 times more (10 files, every one decided by that margin), see [`lsb_is_smoother`].
|
||||
//! - Every other shape of those and of the other private values (32770 with fewer bytes: NX30, NX300, NX2000,
|
||||
//! EK-GN120; 32772: NX mini, NX3000, NX3300; 32773: NX1, NX500) is a compressed coding: reported as unsupported.
|
||||
//! - The maker note is a plain IFD whose value offsets count from the start of the note, not of the TIFF. Several
|
||||
//! vectors are stored offset by `EncryptionKey`, 11 integers in every file (element `i` of a vector uses key
|
||||
//! `i mod 11`): `WB_RGGBLevelsUncorrected` and `WB_RGGBLevelsBlack` hold `value + key`, `RawData` holds
|
||||
//! `value - key`, found by trying both signs and keeping the one that gives plausible numbers (green levels
|
||||
//! 4096 in all but one file, saturation levels 4095 or 16383, black levels 0 to 3, the dark-column level).
|
||||
//! - `SensorAreas`: the second rectangle (left, top, right, bottom) is the camera JPEG's framing (its size equals the
|
||||
//! JPEG's in every file but WB2000's), used as the active area. Registered against the JPEG its centre agrees to
|
||||
//! within 2 pixels (NX10 family; the NX20 family's JPEG is lens-corrected, edges differ).
|
||||
//! - The Exif picture size is smaller than the framing in one direction only on the WB2000 (16:9 from a 4:3 sensor):
|
||||
//! the JPEG is the centre of the framing (registration: top offset 366 = framing top 22 + (2736 - 2048) / 2).
|
||||
//! - `RawData`: four black levels, then saturation levels (R, G1, G2, B twice). The EX1 saturates at 15881, not at its
|
||||
//! stated 16383 (85786 samples sit on the plateau), so a plateau below the stated level wins.
|
||||
//! - Black: `WB_RGGBLevelsBlack` when it states one (NX5, NX10, NX11: 2 to 3, the right-hand dark columns measure
|
||||
//! 3.5 to 4.5); otherwise the dark rows or columns outside the framing (EX1: eight rows at 2.3); otherwise 0.
|
||||
//! - `WB_RGGBLevelsUncorrected` divided by its mean green is the as-shot gain: the renders of the 12 files match the
|
||||
//! camera JPEG at median ΔE76 2.2 to 7.6 (10.7 on the backlit WB2000 sample, 5.7 after registration).
|
||||
use super::{black_from_columns, white_from_data};
|
||||
use crate::unpack::{read_u16s, unpack_lsb, unpack_msb};
|
||||
use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result};
|
||||
use lightcraft_geom::Orientation;
|
||||
use lightcraft_tiff::image::chunk_bytes;
|
||||
use lightcraft_tiff::{ByteOrder, Ifd, Tiff, tags as t};
|
||||
use rayon::prelude::*;
|
||||
|
||||
const SENSOR_AREAS: u16 = 0xa010;
|
||||
const ENCRYPTION_KEY: u16 = 0xa020;
|
||||
const WB_UNCORRECTED: u16 = 0xa021;
|
||||
const WB_BLACK: u16 = 0xa028;
|
||||
const RAW_DATA: u16 = 0xa048;
|
||||
|
||||
/// The private `Compression` values of the raw image IFD.
|
||||
const COMPRESSIONS: std::ops::RangeInclusive<u16> = 32769..=32773;
|
||||
|
||||
/// The maker-note IFD (offsets relative to the note's own start).
|
||||
fn maker_note(bytes: &[u8], tiff: &Tiff) -> Option<Ifd> {
|
||||
let e = tiff.exif()?.get(t::MAKER_NOTE)?;
|
||||
let opts = lightcraft_tiff::ParseOptions { max_ifds: 4, max_depth: 1, follow_children: false, ..Default::default() };
|
||||
lightcraft_tiff::parse_ifd_at(bytes, e.offset, tiff.order, e.offset, false, &opts).ok().map(|(i, _)| i)
|
||||
}
|
||||
|
||||
/// A key-offset vector: the entries of `tag` shifted by the maker note's key (`sign` is +1 or -1 times the key).
|
||||
fn keyed(mn: &Ifd, tag: u16, sign: i64) -> Option<Vec<i64>> {
|
||||
let key = mn.u64s(ENCRYPTION_KEY).filter(|k| k.len() == 11)?;
|
||||
let vals = mn.f64s(tag)?;
|
||||
// 32-bit entries: the stored value wraps modulo 2^32 (a negative number stored as a LONG)
|
||||
Some(vals.iter().enumerate().map(|(i, &v)| i64::from((v as i64).wrapping_add(sign * key[i % 11] as i64) as i32)).collect())
|
||||
}
|
||||
|
||||
/// The raw image's IFD: the one with a private compression value.
|
||||
fn raw_ifd(tiff: &Tiff) -> Option<&Ifd> {
|
||||
tiff.all_ifds().into_iter().find(|i| i.u16(t::COMPRESSION).is_some_and(|c| COMPRESSIONS.contains(&c)) && i.contains(t::IMAGE_WIDTH))
|
||||
}
|
||||
|
||||
/// How the samples of an IFD are stored, when uncompressed.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum Coding {
|
||||
/// One little-endian 16-bit word per sample.
|
||||
Words,
|
||||
/// 12-bit samples, two in three bytes: read MSB-first (the NX5, NX10 and NX11 files) or LSB-first (the later
|
||||
/// bodies' files); the file does not say which, the smoother image does ([`lsb_is_smoother`]).
|
||||
Packed12,
|
||||
}
|
||||
|
||||
fn coding(compression: u16, w: usize, h: usize, strip: u64) -> Option<Coding> {
|
||||
let n = (w as u64).checked_mul(h as u64)?;
|
||||
match compression {
|
||||
32769 if strip == n.checked_mul(2)? => Some(Coding::Words),
|
||||
32770 if w.is_multiple_of(2) && strip == n.checked_mul(12)? / 8 => Some(Coding::Packed12),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
|
||||
let tiff = Tiff::parse(bytes)?;
|
||||
let ifd0 = tiff.ifds.first().ok_or_else(|| RawError::Corrupt("SRW without IFDs".into()))?;
|
||||
let raw = raw_ifd(&tiff).ok_or_else(|| RawError::Unsupported("Samsung raw image not found".into()))?;
|
||||
let info = raw.image()?;
|
||||
let (w, h) = (info.width as usize, info.height as usize);
|
||||
let n = w.checked_mul(h).filter(|n| *n > 0 && *n <= crate::MAX_SAMPLES).ok_or(RawError::Limit("image too large"))?;
|
||||
let chunks = info.chunks(bytes.len() as u64);
|
||||
let strip: u64 = chunks.iter().map(|c| c.len).sum();
|
||||
let Some(coding) = coding(info.compression, w, h, strip).filter(|_| chunks.len() == 1) else {
|
||||
return Err(RawError::Unsupported(format!(
|
||||
"Samsung compressed raw (compression {}, {:.2} stored bits per sample)",
|
||||
info.compression,
|
||||
strip as f64 * 8.0 / n as f64
|
||||
)));
|
||||
};
|
||||
let data = if mode == Mode::Full {
|
||||
let src = chunk_bytes(bytes, &chunks[0]).ok_or_else(|| RawError::Corrupt("SRW strip outside file".into()))?;
|
||||
let mut d = vec![0u16; n];
|
||||
match coding {
|
||||
Coding::Words => read_u16s(src, ByteOrder::Little, &mut d),
|
||||
Coding::Packed12 => {
|
||||
let stride = w / 2 * 3;
|
||||
let unpack: fn(&[u8], u32, &mut [u16]) = if lsb_is_smoother(src, w, h) { unpack_lsb } else { unpack_msb };
|
||||
d.par_chunks_mut(w).zip(src.par_chunks(stride)).for_each(|(row, s)| unpack(s, 12, row));
|
||||
}
|
||||
}
|
||||
RawData::U16(d)
|
||||
} else {
|
||||
RawData::U16(Vec::new())
|
||||
};
|
||||
|
||||
let mn = maker_note(bytes, &tiff);
|
||||
// the camera JPEG's framing (left, top, right, bottom), when it lies inside the stored array
|
||||
let active = mn
|
||||
.as_ref()
|
||||
.and_then(|m| m.u64s(SENSOR_AREAS))
|
||||
.filter(|v| v.len() == 8)
|
||||
.map(|v| (v[4] as usize, v[5] as usize, v[6] as usize, v[7] as usize))
|
||||
.filter(|&(l, tp, r, b)| l < r && tp < b && r <= w && b <= h)
|
||||
.map_or(Rect::new(0, 0, w, h), |(l, tp, r, b)| Rect::new(l, tp, r - l, b - tp));
|
||||
let cfa = match (raw.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), raw.bytes(t::CFA_PATTERN_EP)) {
|
||||
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() },
|
||||
_ => return Err(RawError::Unsupported("Samsung raw without a 2x2 CFAPattern".into())),
|
||||
};
|
||||
// saturation levels and black levels (R, G1, G2, B), in the file's own words
|
||||
let levels = mn.as_ref().and_then(|m| keyed(m, RAW_DATA, 1)).filter(|v| v.len() == 12);
|
||||
let RawData::U16(samples) = &data else { return Err(RawError::Unsupported("float SRW".into())) };
|
||||
// saturation: the file's level, or lower where the samples pile up (EX1 clips at 15882 of its stated 16383)
|
||||
let stated = levels.as_ref().filter(|v| v[4..].iter().all(|&x| x == v[4]) && (255..=65535).contains(&v[4])).map(|v| v[4] as f32);
|
||||
let bits = stated.map_or((info.bits() as u32).clamp(8, 16), |s| 32 - (s as u32).leading_zeros());
|
||||
let plateau = (!samples.is_empty()).then(|| white_from_data(&active_rows(samples, w, active), bits));
|
||||
let white = match (stated, plateau) {
|
||||
(Some(s), Some(p)) => s.min(p),
|
||||
(Some(s), None) => s,
|
||||
(None, Some(p)) => p,
|
||||
(None, None) => 4095.0,
|
||||
};
|
||||
let tagged_black =
|
||||
mn.as_ref().and_then(|m| keyed(m, WB_BLACK, -1)).filter(|v| v.len() == 4 && v.iter().all(|&x| (0..=u16::MAX as i64).contains(&x)));
|
||||
let black = match tagged_black {
|
||||
Some(v) if v.iter().any(|&x| x > 0) => {
|
||||
// RGGB-ordered levels -> per CFA position at the active area origin
|
||||
let a = cfa.shifted(active.x, active.y);
|
||||
let values = a.pattern.iter().map(|&c| [v[0], (v[1] + v[2]) / 2, v[3]][c as usize] as f32).collect();
|
||||
BlackLevel { repeat_rows: 2, repeat_cols: 2, values, ..Default::default() }
|
||||
}
|
||||
// the file states no black level: measure the masked band around the active area, when there is one
|
||||
_ => masked_black(samples, w, h, active, white),
|
||||
};
|
||||
// as-shot white balance: gains with the greens near 4096 (unity)
|
||||
let wb = mn.as_ref().and_then(|m| keyed(m, WB_UNCORRECTED, -1)).filter(|v| v.len() == 4 && v.iter().all(|&x| x > 0)).map(|v| {
|
||||
let g = (v[1] + v[2]) as f64 / 2.0;
|
||||
[(v[0] as f64 / g) as f32, 1.0, (v[3] as f64 / g) as f32]
|
||||
});
|
||||
let mut metadata = lightcraft_meta::from_tiff(&tiff);
|
||||
let crop = picture_crop(&tiff, active);
|
||||
metadata.width = Some(crop.width as u32);
|
||||
metadata.height = Some(crop.height as u32);
|
||||
let img = RawImage {
|
||||
format: RawFormat::Srw,
|
||||
width: w,
|
||||
height: h,
|
||||
cpp: 1,
|
||||
data,
|
||||
cfa: Some(cfa),
|
||||
bits,
|
||||
black,
|
||||
white: vec![white],
|
||||
active_area: active,
|
||||
crop,
|
||||
orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)),
|
||||
color: ColorData::default(),
|
||||
wb_multipliers: wb,
|
||||
linearized: false,
|
||||
opcodes: OpcodeLists::default(),
|
||||
metadata,
|
||||
};
|
||||
img.validate_for(mode)?;
|
||||
Ok(img)
|
||||
}
|
||||
|
||||
/// Which bit order unpacks 12-bit samples of `w` x `h` (in `src`, `w / 2 * 3` bytes per row) into the image: in
|
||||
/// the right order neighbouring samples of one colour differ by little, in the wrong one by about the whole range.
|
||||
/// Compares the summed absolute differences of samples two apart over 64 rows from the middle of the frame.
|
||||
fn lsb_is_smoother(src: &[u8], w: usize, h: usize) -> bool {
|
||||
let stride = w / 2 * 3;
|
||||
let (mut msb, mut lsb) = (0u64, 0u64);
|
||||
let (mut a, mut b) = (vec![0u16; w], vec![0u16; w]);
|
||||
for y in (h / 5..h * 4 / 5).step_by((h * 3 / 5 / 64).max(1)) {
|
||||
let Some(row) = y.checked_mul(stride).and_then(|s| src.get(s..s + stride)) else { break };
|
||||
unpack_msb(row, 12, &mut a);
|
||||
unpack_lsb(row, 12, &mut b);
|
||||
let spread = |v: &[u16]| v.windows(3).map(|p| u64::from(p[0].abs_diff(p[2]))).sum::<u64>();
|
||||
msb += spread(&a);
|
||||
lsb += spread(&b);
|
||||
}
|
||||
lsb < msb
|
||||
}
|
||||
|
||||
/// The default crop: the whole framing, or, when the Exif picture size is the framing cut in one direction only
|
||||
/// (a 16:9 picture from a 4:3 sensor), the centred part of that size.
|
||||
fn picture_crop(tiff: &Tiff, active: Rect) -> Rect {
|
||||
let whole = Rect::new(0, 0, active.width, active.height);
|
||||
let Some(exif) = tiff.exif() else { return whole };
|
||||
let (Some(pw), Some(ph)) = (exif.u32(t::PIXEL_X_DIMENSION), exif.u32(t::PIXEL_Y_DIMENSION)) else { return whole };
|
||||
let (pw, ph) = (pw as usize, ph as usize);
|
||||
if pw == 0 || ph == 0 || pw > active.width || ph > active.height || (pw < active.width && ph < active.height) {
|
||||
return whole;
|
||||
}
|
||||
Rect::new((active.width - pw) / 2, (active.height - ph) / 2, pw, ph)
|
||||
}
|
||||
|
||||
/// Every third row of the active area (the saturation statistics need no more).
|
||||
fn active_rows(d: &[u16], w: usize, a: Rect) -> Vec<u16> {
|
||||
(a.y..a.y + a.height).step_by(3).filter_map(|y| d.get(y * w + a.x..y * w + a.x + a.width)).flatten().copied().collect()
|
||||
}
|
||||
|
||||
/// Black level from rows or columns outside the active area that are dark (mean below 1.5% of `white`): the
|
||||
/// longest run of dark lines on the right, else the left, below, above; at least four lines, the two outermost
|
||||
/// rows and columns of the array left out. Uniform 0 when there is no such run.
|
||||
fn masked_black(d: &[u16], w: usize, h: usize, a: Rect, white: f32) -> BlackLevel {
|
||||
if d.is_empty() {
|
||||
return BlackLevel::uniform(0.0);
|
||||
}
|
||||
let limit = f64::from(white) * 0.015;
|
||||
let col_mean = |x: usize| {
|
||||
(a.y..a.y + a.height).step_by(7).filter_map(|y| d.get(y * w + x)).map(|&v| f64::from(v)).sum::<f64>() / a.height.div_ceil(7) as f64
|
||||
};
|
||||
let row_mean =
|
||||
|y: usize| (a.x..a.x + a.width).step_by(7).filter_map(|x| d.get(y * w + x)).map(|&v| f64::from(v)).sum::<f64>() / a.width.div_ceil(7) as f64;
|
||||
// the longest run of consecutive dark lines within `lines`
|
||||
let longest = |lines: std::ops::Range<usize>, mean: &dyn Fn(usize) -> f64| {
|
||||
let (mut best, mut start) = (0..0, lines.start);
|
||||
for i in lines.clone() {
|
||||
if mean(i) >= limit {
|
||||
start = i + 1;
|
||||
} else if i + 1 - start > best.len() {
|
||||
best = start..i + 1;
|
||||
}
|
||||
}
|
||||
best
|
||||
};
|
||||
let cols = a.y..a.y + a.height;
|
||||
let rows = a.x..a.x + a.width;
|
||||
let right = longest(a.x + a.width..w.saturating_sub(2), &col_mean);
|
||||
let left = longest(2..a.x, &col_mean);
|
||||
let below = longest(a.y + a.height..h.saturating_sub(2), &row_mean);
|
||||
let above = longest(2..a.y, &row_mean);
|
||||
if right.len() >= 4 {
|
||||
black_from_columns(d, w, right, cols, a)
|
||||
} else if left.len() >= 4 {
|
||||
black_from_columns(d, w, left, cols, a)
|
||||
} else if below.len() >= 4 {
|
||||
black_from_columns(d, w, rows, below, a)
|
||||
} else if above.len() >= 4 {
|
||||
black_from_columns(d, w, rows, above, a)
|
||||
} else {
|
||||
BlackLevel::uniform(0.0)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{decode, probe, probe_info};
|
||||
use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value};
|
||||
|
||||
const KEY: [u32; 11] = [305, 72, 737, 456, 282, 307, 519, 724, 13, 505, 193];
|
||||
|
||||
/// A maker note in the Samsung layout: a big-endian IFD of LONG vectors, offsets relative to the note's start.
|
||||
fn note(entries: &[(u16, Vec<u32>)]) -> Vec<u8> {
|
||||
let mut head = Vec::new();
|
||||
let mut tail = Vec::new();
|
||||
head.extend_from_slice(&(entries.len() as u16).to_be_bytes());
|
||||
let values_at = 2 + 12 * entries.len() + 4;
|
||||
for (tag, v) in entries {
|
||||
head.extend_from_slice(&tag.to_be_bytes());
|
||||
head.extend_from_slice(&4u16.to_be_bytes());
|
||||
head.extend_from_slice(&(v.len() as u32).to_be_bytes());
|
||||
if v.len() == 1 {
|
||||
head.extend_from_slice(&v[0].to_be_bytes());
|
||||
} else {
|
||||
head.extend_from_slice(&((values_at + tail.len()) as u32).to_be_bytes());
|
||||
tail.extend(v.iter().flat_map(|x| x.to_be_bytes()));
|
||||
}
|
||||
}
|
||||
head.extend_from_slice(&[0; 4]);
|
||||
head.extend(tail);
|
||||
head
|
||||
}
|
||||
|
||||
/// Stored form of a vector the way the file keeps it: `plain + key` (`sign` 1) or `plain - key` (`sign` -1).
|
||||
fn stored(plain: &[i64], sign: i64) -> Vec<u32> {
|
||||
plain.iter().enumerate().map(|(i, &p)| (p + sign * i64::from(KEY[i % 11])) as u32).collect()
|
||||
}
|
||||
|
||||
fn pack12(v: &[u16]) -> Vec<u8> {
|
||||
v.chunks(2).flat_map(|p| [(p[0] >> 4) as u8, ((p[0] & 15) << 4) as u8 | (p[1] >> 8) as u8, p[1] as u8]).collect()
|
||||
}
|
||||
|
||||
fn le_words(v: &[u16]) -> Vec<u8> {
|
||||
v.iter().flat_map(|x| x.to_le_bytes()).collect()
|
||||
}
|
||||
|
||||
struct Spec {
|
||||
w: u32,
|
||||
h: u32,
|
||||
compression: u16,
|
||||
strip: Vec<u8>,
|
||||
pattern: Option<[u8; 4]>,
|
||||
note: Option<Vec<u8>>,
|
||||
picture: Option<(u32, u32)>,
|
||||
}
|
||||
|
||||
fn srw(s: Spec) -> Vec<u8> {
|
||||
let mut raw = IfdBuilder::new();
|
||||
raw.set(t::NEW_SUBFILE_TYPE, Value::Long(vec![0]));
|
||||
raw.set(t::IMAGE_WIDTH, Value::Long(vec![s.w]));
|
||||
raw.set(t::IMAGE_LENGTH, Value::Long(vec![s.h]));
|
||||
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![12]));
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![s.compression]));
|
||||
raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1]));
|
||||
if let Some(p) = s.pattern {
|
||||
raw.set(t::CFA_REPEAT_PATTERN_DIM, Value::Short(vec![2, 2]));
|
||||
raw.set(t::CFA_PATTERN_EP, Value::Byte(p.to_vec()));
|
||||
}
|
||||
raw.set_image(ImageData::Strips { rows_per_strip: s.h, strips: vec![s.strip] });
|
||||
let mut ifd0 = IfdBuilder::new();
|
||||
ifd0.set(t::MAKE, Value::Ascii("SAMSUNG".into()));
|
||||
ifd0.set(t::MODEL, Value::Ascii("NX0".into()));
|
||||
let mut exif = IfdBuilder::new();
|
||||
if let Some(n) = s.note {
|
||||
exif.set(t::MAKER_NOTE, Value::Undefined(n));
|
||||
}
|
||||
if let Some((pw, ph)) = s.picture {
|
||||
exif.set(t::PIXEL_X_DIMENSION, Value::Long(vec![pw]));
|
||||
exif.set(t::PIXEL_Y_DIMENSION, Value::Long(vec![ph]));
|
||||
}
|
||||
ifd0.set_child(t::EXIF_IFD, exif);
|
||||
ifd0.add_sub_ifd(raw);
|
||||
TiffWriter::new(lightcraft_tiff::ByteOrder::Big, false).write(&[ifd0]).unwrap()
|
||||
}
|
||||
|
||||
/// The camera-stated description every test file carries: JPEG framing, white balance, levels.
|
||||
fn full_note(area: [u32; 4], white: i64, black: i64) -> Vec<u8> {
|
||||
let mut levels = vec![0i64; 4];
|
||||
levels.extend([white; 8]);
|
||||
note(&[
|
||||
(SENSOR_AREAS, vec![0, 0, 64, 32, area[0], area[1], area[2], area[3]]),
|
||||
(ENCRYPTION_KEY, KEY.to_vec()),
|
||||
(WB_UNCORRECTED, stored(&[8192, 4096, 4096, 6144], 1)),
|
||||
(WB_BLACK, stored(&[black; 4], 1)),
|
||||
(RAW_DATA, stored(&levels, -1)),
|
||||
])
|
||||
}
|
||||
|
||||
/// A smooth gradient with a little texture, `max` the first value that is not reached.
|
||||
fn ramp(w: u32, h: u32, max: u32) -> Vec<u16> {
|
||||
(0..w * h).map(|i| ((1000 + (i % w) * 7 + (i / w) * 5 + (i * 13) % 9) % max) as u16).collect()
|
||||
}
|
||||
|
||||
fn pack12_lsb(v: &[u16]) -> Vec<u8> {
|
||||
v.chunks(2).flat_map(|p| [p[0] as u8, ((p[0] >> 8) as u8) | ((p[1] & 15) << 4) as u8, (p[1] >> 4) as u8]).collect()
|
||||
}
|
||||
|
||||
/// Known answer: the samples, layout, framing, levels and white balance all come from the file's own words.
|
||||
#[test]
|
||||
fn packed_12_bit_samples_decode_with_the_maker_note() {
|
||||
let v = ramp(64, 32, 4096);
|
||||
let bytes = srw(Spec {
|
||||
w: 64,
|
||||
h: 32,
|
||||
compression: 32770,
|
||||
strip: pack12(&v),
|
||||
pattern: Some([2, 1, 1, 0]),
|
||||
note: Some(full_note([6, 4, 58, 30], 4095, 3)),
|
||||
picture: None,
|
||||
});
|
||||
assert_eq!(probe(&bytes), Some(RawFormat::Srw));
|
||||
let r = decode(&bytes).unwrap();
|
||||
assert_eq!((r.format, r.width, r.height, r.bits), (RawFormat::Srw, 64, 32, 12));
|
||||
assert_eq!(r.data, RawData::U16(v));
|
||||
assert_eq!(r.cfa.as_ref().unwrap().name(), "BGGR");
|
||||
assert_eq!(r.active_area, Rect::new(6, 4, 52, 26));
|
||||
assert_eq!(r.crop, Rect::new(0, 0, 52, 26));
|
||||
assert_eq!((r.black.mean(), r.white.clone()), (3.0, vec![4095.0]));
|
||||
assert_eq!(r.wb_multipliers, Some([2.0, 1.0, 1.5]));
|
||||
// headers only: everything but the samples
|
||||
let i = probe_info(&bytes).unwrap();
|
||||
assert_eq!((i.active_area, i.wb_multipliers, i.bits), (Rect::new(6, 4, 52, 26), Some([2.0, 1.0, 1.5]), 12));
|
||||
}
|
||||
|
||||
/// The later bodies pack the same 12-bit samples LSB-first; which of the two orders a file uses is found from
|
||||
/// the samples (the wrong order makes a noisy image), not from the model.
|
||||
#[test]
|
||||
fn lsb_first_packing_is_found_from_the_samples() {
|
||||
let v = ramp(64, 32, 4096);
|
||||
for (strip, name) in [(pack12(&v), "msb"), (pack12_lsb(&v), "lsb")] {
|
||||
let bytes = srw(Spec { w: 64, h: 32, compression: 32770, strip, pattern: Some([1, 0, 2, 1]), note: None, picture: None });
|
||||
assert_eq!(decode(&bytes).unwrap().data, RawData::U16(v.clone()), "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A picture size narrower than the framing in one direction only (16:9 from a 4:3 sensor) is the framing's
|
||||
/// centre; any other size is ignored.
|
||||
#[test]
|
||||
fn the_exif_picture_size_centres_a_one_direction_crop() {
|
||||
let v = ramp(64, 32, 4096);
|
||||
let file = |picture| {
|
||||
srw(Spec {
|
||||
w: 64,
|
||||
h: 32,
|
||||
compression: 32770,
|
||||
strip: pack12(&v),
|
||||
pattern: Some([2, 1, 1, 0]),
|
||||
note: Some(full_note([6, 4, 58, 30], 4095, 0)),
|
||||
picture,
|
||||
})
|
||||
};
|
||||
// framing 52 x 26
|
||||
assert_eq!(decode(&file(Some((52, 18)))).unwrap().crop, Rect::new(0, 4, 52, 18));
|
||||
assert_eq!(decode(&file(Some((40, 26)))).unwrap().crop, Rect::new(6, 0, 40, 26));
|
||||
let r = decode(&file(Some((52, 18)))).unwrap();
|
||||
assert_eq!((r.metadata.width, r.metadata.height), (Some(52), Some(18)));
|
||||
for other in [None, Some((52, 26)), Some((40, 18)), Some((60, 18)), Some((52, 30)), Some((0, 18))] {
|
||||
assert_eq!(decode(&file(other)).unwrap().crop, Rect::new(0, 0, 52, 26), "{other:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// 16-bit words are little-endian although the file is big-endian; a saturation plateau below the stated
|
||||
/// level wins; dark rows below the framing give the black level when the file states none.
|
||||
#[test]
|
||||
fn word_samples_decode_and_measure_what_the_file_does_not_say() {
|
||||
let (w, h) = (64u32, 40u32);
|
||||
let mut v: Vec<u16> = (0..w * h).map(|i| 3000 + (i * 91 % 5000) as u16).collect();
|
||||
// saturation: a plateau at 15882 inside the framing (rows 4..28), two rows of picture beyond it, dark rows 30..40
|
||||
for i in 0..150usize {
|
||||
v[(4 + i % 24) * 64 + 8 + i / 24] = 15882;
|
||||
}
|
||||
for x in 0..64 * 10 {
|
||||
v[30 * 64 + x] = 80;
|
||||
}
|
||||
let bytes = srw(Spec {
|
||||
w,
|
||||
h,
|
||||
compression: 32769,
|
||||
strip: le_words(&v),
|
||||
pattern: Some([0, 1, 1, 2]),
|
||||
note: Some(full_note([0, 4, 64, 28], 16383, 0)),
|
||||
picture: None,
|
||||
});
|
||||
let r = decode(&bytes).unwrap();
|
||||
assert_eq!(r.data, RawData::U16(v));
|
||||
assert_eq!(r.bits, 14);
|
||||
assert!((15800.0..=15882.0).contains(&r.white[0]), "{:?}", r.white);
|
||||
assert_eq!(r.black.mean(), 80.0);
|
||||
assert_eq!(r.cfa.unwrap().name(), "RGGB");
|
||||
}
|
||||
|
||||
/// Files whose strip is not exactly the uncompressed size (or that use another private value) are compressed
|
||||
/// codings: reported as such, still recognised as SRW.
|
||||
#[test]
|
||||
fn compressed_codings_are_reported_not_misread() {
|
||||
let plain = |compression, strip: Vec<u8>| {
|
||||
srw(Spec { w: 64, h: 32, compression, strip, pattern: Some([2, 1, 1, 0]), note: Some(full_note([0, 0, 64, 32], 4095, 0)), picture: None })
|
||||
};
|
||||
for bytes in [
|
||||
plain(32770, vec![7; 64 * 32 * 12 / 8 - 3]),
|
||||
plain(32772, vec![7; 4000]),
|
||||
plain(32773, pack12(&ramp(64, 32, 4096))),
|
||||
plain(32769, vec![0; 64 * 32]),
|
||||
] {
|
||||
assert_eq!(probe(&bytes), Some(RawFormat::Srw));
|
||||
let Err(RawError::Unsupported(why)) = decode(&bytes) else { panic!("expected Unsupported") };
|
||||
assert!(why.contains("compressed"), "{why}");
|
||||
assert!(probe_info(&bytes).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
/// Without a maker note the samples still decode: whole array, no white balance, depth from the TIFF tags.
|
||||
#[test]
|
||||
fn a_file_without_a_maker_note_decodes_with_defaults() {
|
||||
let v = ramp(64, 32, 4096);
|
||||
let bytes = srw(Spec { w: 64, h: 32, compression: 32770, strip: pack12(&v), pattern: Some([1, 0, 2, 1]), note: None, picture: None });
|
||||
let r = decode(&bytes).unwrap();
|
||||
assert_eq!(r.data, RawData::U16(v));
|
||||
assert_eq!((r.active_area, r.wb_multipliers, r.black.mean()), (Rect::new(0, 0, 64, 32), None, 0.0));
|
||||
assert_eq!(r.cfa.unwrap().name(), "GRBG");
|
||||
assert_eq!(r.bits, 12);
|
||||
}
|
||||
|
||||
/// A raw IFD that does not say how its mosaic is laid out is not guessed at.
|
||||
#[test]
|
||||
fn a_file_without_a_cfa_pattern_is_unsupported() {
|
||||
let bytes = srw(Spec { w: 64, h: 32, compression: 32770, strip: pack12(&ramp(64, 32, 4096)), pattern: None, note: None, picture: None });
|
||||
assert!(matches!(decode(&bytes), Err(RawError::Unsupported(w)) if w.contains("CFAPattern")));
|
||||
}
|
||||
|
||||
/// A framing rectangle outside the array, or reversed, is ignored; truncated files fail without panicking.
|
||||
#[test]
|
||||
fn hostile_framing_and_truncation_do_not_panic() {
|
||||
let v = ramp(64, 32, 4096);
|
||||
for area in [[0, 0, 65, 32], [10, 0, 10, 32], [0, 40, 64, 20]] {
|
||||
let bytes = srw(Spec {
|
||||
w: 64,
|
||||
h: 32,
|
||||
compression: 32770,
|
||||
strip: pack12(&v),
|
||||
pattern: Some([2, 1, 1, 0]),
|
||||
note: Some(full_note(area, 4095, 0)),
|
||||
picture: None,
|
||||
});
|
||||
assert_eq!(decode(&bytes).unwrap().active_area, Rect::new(0, 0, 64, 32));
|
||||
}
|
||||
let bytes = srw(Spec {
|
||||
w: 64,
|
||||
h: 32,
|
||||
compression: 32770,
|
||||
strip: pack12(&v),
|
||||
pattern: Some([2, 1, 1, 0]),
|
||||
note: Some(full_note([0, 0, 64, 32], 4095, 0)),
|
||||
picture: None,
|
||||
});
|
||||
for cut in [10, bytes.len() / 2, bytes.len() - 20] {
|
||||
let _ = decode(&bytes[..cut]);
|
||||
let _ = probe_info(&bytes[..cut]);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -528,6 +528,63 @@ fn corpus_orf_cfa_patterns() {
|
||||
eprintln!("ORF CFA layouts checked on {seen} files");
|
||||
}
|
||||
|
||||
/// Samsung SRW (`crates/raw/src/vendor/srw.rs`): the 12-bit packings of the NX10 and NX20 generations (opposite bit
|
||||
/// orders, told apart from the samples) and the 16-bit words of the EX1 and WB2000 decode to the image of the camera's own
|
||||
/// JPEG, with the layout, framing, levels and white balance the file's maker note states.
|
||||
#[test]
|
||||
fn corpus_samsung_srw() {
|
||||
let dir = corpus_root().join("raw");
|
||||
// (file, bits, active area width x height, crop width x height, CFA layout, JPEG shows the whole active area)
|
||||
let cases = [
|
||||
("srw-samsung-nx10.srw", 12, (4592, 3056), (4592, 3056), "BGGR", true),
|
||||
("srw-samsung-nx20.srw", 12, (5472, 3648), (5472, 3648), "GRBG", true),
|
||||
("srw-samsung-ex1.srw", 14, (3648, 2736), (3648, 2736), "RGGB", true),
|
||||
];
|
||||
let mut seen = 0;
|
||||
for (name, bits, active, crop, cfa, whole) in cases {
|
||||
let Ok(bytes) = std::fs::read(dir.join(name)) else {
|
||||
eprintln!("skip: {name} absent");
|
||||
continue;
|
||||
};
|
||||
let img = decode(&bytes).unwrap_or_else(|e| panic!("{name}: {e}"));
|
||||
assert_eq!(img.format, RawFormat::Srw, "{name}");
|
||||
assert_eq!(img.bits, bits, "{name}: bits");
|
||||
assert_eq!((img.active_area.width, img.active_area.height), active, "{name}: active area");
|
||||
assert_eq!((img.crop.width, img.crop.height), crop, "{name}: crop");
|
||||
assert_eq!(img.cfa.as_ref().map(|c| c.name()).as_deref(), Some(cfa), "{name}: CFA layout");
|
||||
let wb = img.wb_multipliers.unwrap_or_else(|| panic!("{name}: no as-shot white balance"));
|
||||
assert!(wb[0] > 1.0 && wb[1] == 1.0 && wb[2] > 0.5, "{name}: white balance {wb:?}");
|
||||
assert!(img.white_at(0) > (1u32 << bits) as f32 * 0.9 && img.white_at(0) <= ((1u32 << bits) - 1) as f32, "{name}: white {}", img.white_at(0));
|
||||
assert!(img.black.mean() < 8.0, "{name}: black {}", img.black.mean());
|
||||
// the mosaic inside the framing holds no marker or padding values above the saturation level
|
||||
let lightcraft_raw::RawData::U16(d) = &img.data else { panic!("float data") };
|
||||
let a = img.active_area;
|
||||
let above = (a.y..a.y + a.height)
|
||||
.map(|y| d[y * img.width + a.x..y * img.width + a.x + a.width].iter().filter(|&&v| f32::from(v) > ((1u32 << bits) - 1) as f32).count())
|
||||
.sum::<usize>();
|
||||
assert_eq!(above, 0, "{name}: samples above the saturation level in the active area");
|
||||
if whole {
|
||||
let jpeg = embedded_preview(&bytes).unwrap_or_else(|| panic!("{name}: no embedded preview"));
|
||||
let (gw, gh) = (24, 18);
|
||||
let (sites, jpeg) = (raw_sites(&img, gw, gh), jpeg_cells(&jpeg, gw, gh));
|
||||
let layout = img.cfa.clone().unwrap_or_else(|| panic!("{name}: no CFA"));
|
||||
let green: Vec<f64> = sites.iter().map(|s| (0..4).filter(|&i| layout.pattern[i] == 1).map(|i| s[i]).sum::<f64>()).collect();
|
||||
let rho = correlation(&ranks(&green), &ranks(&jpeg.iter().map(|j| j[1]).collect::<Vec<_>>()));
|
||||
let agree = |l: &str| chroma_agreement(&sites, &jpeg, &lightcraft_raw::Cfa::bayer(l).unwrap().pattern);
|
||||
let best_other = ["RGGB", "GRBG", "GBRG", "BGGR"].into_iter().filter(|l| *l != cfa).map(agree).fold(f64::MIN, f64::max);
|
||||
let own = agree(cfa);
|
||||
eprintln!(
|
||||
"{name:26} {}x{} {bits}-bit: green rank correlation {rho:.3}, colour agreement {own:.3} (others <= {best_other:.3})",
|
||||
img.width, img.height
|
||||
);
|
||||
assert!(rho > 0.9, "{name}: rank correlation with the camera JPEG {rho}");
|
||||
assert!(own > 0.5 && own > best_other, "{name}: colours follow the JPEG better with another layout ({own} vs {best_other})");
|
||||
}
|
||||
seen += 1;
|
||||
}
|
||||
eprintln!("Samsung SRW checked on {seen} files");
|
||||
}
|
||||
|
||||
/// Reference sensor sums and position-weighted sums from black-box decoding, independent
|
||||
/// of this implementation. These cover complete images, padding, stripe boundaries and
|
||||
/// lossy block quantizer changes; the source files are CC0, fetched by `xtask corpus`.
|
||||
@@ -598,6 +655,7 @@ fn corpus_raws_keep_their_container_and_are_not_thumbnail_shells() {
|
||||
"orf" => RawFormat::Orf,
|
||||
"pef" => RawFormat::Pef,
|
||||
"raf" => RawFormat::Raf,
|
||||
"srw" => RawFormat::Srw,
|
||||
"rw2" | "rwl" | "raw" => RawFormat::Rw2,
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
@@ -669,6 +669,26 @@ fn malformed_jpeg_xl_tiles_are_errors() {
|
||||
assert!(matches!(lightcraft_raw::probe_info(&one(good, 3)), Err(RawError::Corrupt(_))));
|
||||
}
|
||||
|
||||
/// One undecodable tile makes the whole image an error: it is never returned with a black tile.
|
||||
#[test]
|
||||
fn a_jpeg_xl_tile_that_fails_is_an_error_for_the_image() {
|
||||
use lightcraft_raw::RawError;
|
||||
let (w, h) = (32, 16);
|
||||
let px = pattern(w, h, 1, 12);
|
||||
let tile = |x0: usize| -> Vec<u16> { (0..16).flat_map(|y| px[y * w + x0..y * w + x0 + 16].to_vec()).collect() };
|
||||
let two = |second: Vec<u8>| {
|
||||
let mut raw = base_ifd(w, h, 12, 1, true);
|
||||
raw.set(t::COMPRESSION, Value::Short(vec![compression::JPEG_XL]));
|
||||
raw.set_image(ImageData::Tiles { tile_width: 16, tile_height: 16, tiles: vec![jxl_encode(&tile(0), 16, 16, 1, false), second] });
|
||||
dng(raw, ByteOrder::Little)
|
||||
};
|
||||
assert_eq!(decode(&two(jxl_encode(&tile(16), 16, 16, 1, false))).unwrap().data, RawData::U16(px.clone()));
|
||||
// the second tile is cut short; the first one alone is not an image
|
||||
let good = jxl_encode(&tile(16), 16, 16, 1, false);
|
||||
assert!(matches!(decode(&two(good[..good.len() / 2].to_vec())), Err(RawError::Corrupt(_))));
|
||||
assert!(matches!(decode(&two(vec![0xff, 0x0a, 1, 2, 3])), Err(RawError::Corrupt(_))));
|
||||
}
|
||||
|
||||
/// The decoder's memory is bounded by the tile: a grey tile that also codes an alpha channel needs
|
||||
/// about twice what one plane may use, and is an error rather than an allocation the header asks
|
||||
/// for. The same samples without the alpha channel decode.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Sony ARW, Nikon NEF, Panasonic RW2, Fujifilm RAF, Canon CR3/CR2 and Pentax PEF starting look
|
||||
|
||||
ARW, NEF, RW2, RAF, CR3, CR2 and PEF decoding supply sensor data and camera white-balance multipliers, but currently no measured camera colour matrix. Treating camera RGB as linear sRGB produces a dark, muted default rendering. The loader now estimates a file-local starting look from that file's embedded camera JPEG, only when a usable camera matrix is absent. All output pixels still come from the RAW sensor data; there is no JPEG replacement or uniform saturation boost.
|
||||
ARW, NEF, RW2, RAF, CR3, CR2, PEF and SRW decoding supply sensor data and camera white-balance multipliers, but currently no measured camera colour matrix. Treating camera RGB as linear sRGB produces a dark, muted default rendering. The loader now estimates a file-local starting look from that file's embedded camera JPEG, only when a usable camera matrix is absent. All output pixels still come from the RAW sensor data; there is no JPEG replacement or uniform saturation boost.
|
||||
|
||||
## Colour and tone are separate
|
||||
|
||||
@@ -10,7 +10,7 @@ Two thirds of the usable pixels train a ridge-regularised 3×3 **chromaticity**
|
||||
|
||||
The JPEG's colourfulness also depends on brightness, as a camera's per-channel tone curve makes it: relative to a luminance-only curve, shadows were up to 1.4× more colourful and highlights above 0.85 luminance bleached to 0.1–0.3×, so warm sunlit rock rendered cream instead of white (+4 b* in the brightest fifth of the picture). A **chroma curve** (8 nodes over display luminance, `CameraTone::chroma`) is fitted to that ratio, including the bright pixels the colour and tone fits leave out, and applied in the finish stage right after the tone map (CPU and GPU), scaling chroma toward luminance so luminance is kept. It is kept only when it lowers the held-out error; smart previews carry it in their header, and older headers load without one.
|
||||
|
||||
Matrix-only and matrix + table looks are each completed (tone curve, then chroma curve) and compared on the held-out pixels in gamma-encoded display values: in linear values a slightly missed bright rock outweighed a clearly wrong dark shirt. The acceptance gates below still use linear values. Luminance is fitted separately: 32 equal-population bins with robust median knots and pooled-adjacent-violators isotonic regression. The curve preserves black, cannot reverse, and extends beyond observed highlights with a bounded shoulder.
|
||||
Matrix-only and matrix + table looks are each completed (tone curve, then chroma curve) and compared on the held-out pixels in gamma-encoded display values: in linear values a slightly missed bright rock outweighed a clearly wrong dark shirt. The acceptance gates below still use linear values. Luminance is fitted separately, by quantile matching: knot *i* pairs the median of the *i*-th 1/32 of the sorted scene luminances with the median of the *i*-th 1/32 of the sorted JPEG luminances. A tone curve is monotone, so it maps each scene quantile to the same JPEG quantile; this needs no pixel-exact registration and is increasing by construction. The earlier fit (the JPEG's median per scene-luminance bin, then pooled-adjacent-violators isotonic regression) broke on lens-corrected, sharpened JPEGs of busy scenes: neighbouring bins disagreed by more than the gap between them, pooling turned them into flat steps (13 of 31 segments on a CC0 Z 6II forest shot) that rendered as posterised patches with hard edges, and regression toward the mean flattened both ends of the curve (issue #475, also #211, #454). On the 1,781 files of the CC0 archive that render, 384 of the 1,023 accepted fits had at least one flat segment (1,073 in total, up to 14 of 31); with quantile matching 1,017 of the 1,023 have none. The quantile fit is slightly worse by least squares than the conditional medians (median held-out RMS 0.0331 against 0.0330, worst +0.013), and on 7 files (compacts, bridge cameras, early Micro Four Thirds) the look cleared the acceptance gates only with the old curve. The search therefore runs once with the quantile fit (all pixels, away from edges, with and without the camera profile's colour) and, only when no look passes, once more with the earlier conditional-median fit, kept as `fit_tone_conditional`; the gates are the same, so no file loses its look. Those 7 files render exactly as before, and 6 of them still have flat segments (35 in total). The curve preserves black, cannot reverse, and extends beyond observed highlights with a bounded shoulder.
|
||||
|
||||
The matrix is applied to scene-linear sensor pixels. The curve is carried in `SourceInfo` and applied in the pipeline finish stage, after exposure/WB/local edits. It is never baked into decoded luminance or clipped to JPEG precision. CPU and GPU use the same tone LUT. Contrast, whites and blacks adjust the starting look through the display-domain controls. JPEG neutral rendering and the existing DNG colour model are unchanged.
|
||||
|
||||
@@ -85,6 +85,11 @@ An embedded JPEG with neither ICC nor EXIF metadata uses Nikon maker-note `Color
|
||||
|
||||
Synthetic tests cover crop bounds/CFA preservation, colour-space tags, fallback transfer decoding and metadata precedence. A private Nikon Z 8 sample reports an 8256 × 5504 default crop from an 8280 × 5520 sensor. No private sample or Adobe asset is included.
|
||||
|
||||
### Samsung SRW
|
||||
|
||||
Samsung SRW decoding (uncompressed files) supplies the as-shot white-balance multipliers from the maker note and no colour
|
||||
matrix, so SRW takes the same fit, gates and relative white balance.
|
||||
|
||||
### Canon CR2 and Pentax PEF
|
||||
|
||||
CR2 and PEF decode without a camera colour matrix too, but were left out of the fit until issue #310 (CR2 files opening
|
||||
|
||||
+3
-2
@@ -142,7 +142,8 @@ under the UI command that wraps it (a no-op error); the UI command now wins.
|
||||
| LR-IMP-PROFILES | Import profiles & presets | P1 | ✅ | `cmd:file.importPresets`, `cmd:preset.import`, `cmd:profile.import`, `crates/engine/src/preset_import.rs`, `crates/engine/src/preset_luminar.rs` | presets: .lcpreset, XMP, classic .lrtemplate, photos carrying edits ("DNG presets"), Luminar looks (.lmp, .mplumpack collections; sliders with a counterpart), .zip bundles, folders (folder → group), drag & drop; masks carried over; unmapped settings reported (fields that only describe the preset or photo, e.g. `Cluster`, `SortName`, `SupportsAmount2`, the as-shot white, empty Point Color slots, are not); profiles: .cube 3D LUTs; Adobe profile formats deliberately unsupported |
|
||||
| LR-IMP-LOCAL | Work on files in place | P0 | ✅ | `cmd:library.browse`, `cmd:photo.addToLibrary`, `cmd:library.import` (mode add), `cmd:local.addRoot`, `cmd:local.hide`, `cmd:local.restoreHidden`, `crates/engine/src/cmd/browse.rs`, `crates/ui-egui/src/panels/left.rs` (`local_section`), `crates/ui-egui/src/panels/grid.rs` (`folder_header`) | Local: browse Pictures / Desktop / Downloads / Home or any folder without adding it (breadcrumb, Include subfolders, Add N to My Photos); browsed photos stay out of All Photos, albums and counts; folders picked with Browse Folder… (or Keep in Local / Add to Local on a folder row) stay listed across restarts, and browsing a folder inside a listed one highlights it inside that tree (opened down to it, siblings reachable) instead of adding a root; edits go to XMP sidecars; importing promotes them; right-click a top-level location → Remove from Local hides the shortcut only (nothing on disk or in the catalog changes), saved with the UI state, and “Show N hidden locations” (or Browse Folder… on it) brings it back; a folder is one location however its path is spelled (`/` or `\`, trailing separator, `.`/`..`, drive-letter case; `folder_key` in `crates/catalog/src/query.rs`); browsed photos nobody changed are forgotten once their folder has not been browsed for 30 days (Settings → Performance → Local folders, 0 = never; checked when the library opens, or `cmd:library.forgetLocal` with `dryRun`; files and sidecars stay, browsing again brings them back; see `crates/catalog/src/local.rs` for what counts as a change) |
|
||||
| LR-IMP-SIDECAR-SPLIT | Separate XMP sidecar variants | P2 | ⬜ | `cmd:library.xmpPreferences` | sidecar naming option exists (stem/full), no split sidecars |
|
||||
| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs`, `crates/raw/src/jxl.rs` | JPEG, PNG, TIFF (1- to 64-bit integer, 16- to 64-bit float, incl. bilevel and CCITT fax), WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG and DNG 1.7 JPEG XL tiles/strips: lossless ones decode bit-exactly on synthetic CFA and LinearRaw files, lossy ones decode but no real JPEG XL DNG has verified them yet; a single-chunk JPEG XL preview IFD is offered as the embedded preview), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans; `crates/raw/src/vendor/rafc.rs`, full sensor verification on 13 bodies, `docs/raf-compression.md`), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF (uncompressed; colour-filter layout from the file's Exif `CFAPattern`). 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). Lossy type 2 NEF curves of version `0x44 0x40` (D780, D6, Z 5, Z 6, Z 7, Z 7_2) span a quarter of the sample range, read from the table's own point spacing. CR3 lossless CRX Bayer and version 0x100/0x200 C-RAW decodes to sensor pixels with timed white balance, black/white levels and file crop (the shot's aspect ratio from the maker note's `AspectInfo`); full sensor samples match the reference on M50/R100/R8 (`crates/raw/tests/cr3_corpus.rs`, `docs/cr3.md`). Unsupported coding variants retain metadata and use the embedded JPEG when available; HEVC preview decoding remains unsupported. Missing: CR3 unverified CRX variants, compressed ORF, NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), TIFF-based raws whose raw data sits in a private block (Phase One / Leaf IIQ, Canon EOS-1D / 1Ds and Kodak DCS TIFFs: IFD0 is only a reduced RGB image next to a much larger declared picture) are recognised from that structure and import as `preview_only` with the reason, instead of opening as a thumbnail-sized ordinary image (`thumbnail_shell` in `crates/raw/src/lib.rs`), Recognised raws LightCraft cannot decode open as `preview_only` with any usable embedded JPEG, also Canon CRW, Minolta MRW and Sigma X3F (containers recognised by magic, preview found by a structural JPEG scan), Kodak KDC (IFD0 without `ImageWidth`), Leaf MOS and Epson ERF (scan), and any decode failure of a recognised raw, not only an unsupported variant; ones with no JPEG still fail with "without an embedded preview", AVIF decode (HEIC decodes only behind the non-default `heif` feature: `crates/heif`, heic-rs, same policy as PhotoCraft; 8/10-bit, alpha, irot/imir/clap, ICC/EXIF/XMP), palette (indexed-colour) TIFF (refused by the `tiff` crate) |
|
||||
| LR-IMP-FORMATS | Supported formats | P0 | 🟡 | `crates/codecs/src/lib.rs`, `crates/raw/src/lib.rs`, `crates/raw/src/jxl.rs` | JPEG, PNG, TIFF (1- to 64-bit integer, 16- to 64-bit float, incl. bilevel and CCITT fax), WebP, JXL, PSD, GIF, BMP; DNG (incl. lossy / Smart Preview DNG and DNG 1.7 JPEG XL tiles/strips: lossless ones decode bit-exactly on synthetic CFA and LinearRaw files, lossy ones are restored to the declared linear primaries without clipping or gamut mapping, checked on one real file with 108 lossy tiles, and any tile that fails to decode makes the file preview-only; a single-chunk JPEG XL preview IFD is offered as the embedded preview), CR2 (colour-filter layout from the file's `CR2CFAPattern` tag, issue #85), ARW (pre-2017 bodies: white balance and black level from enciphered maker-note / `SR2SubIFD` data, issue #148; downsized Sony lossless YCbCr 4:2:0 / 4:2:2 tiles decoded to linear RGB, verified on private ILCE-7M4 3:2 & 4:3 examples including 16-bit full-resolution export), NEF (uncompressed, lossless and lossy compressed 12/14-bit, `crates/raw/src/vendor/nefc.rs`), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans; `crates/raw/src/vendor/rafc.rs`, full sensor verification on 13 bodies, `docs/raf-compression.md`), RW2 / Leica RWL / Panasonic RAW in every raw format (compressed 4 and 6, the prefix-coded strips of format 8 from the GH6 on, packed 2/5/7, the 16-bit words of the 2005–2007 bodies; black levels, in-camera aspect crops and mapped-out defect pixels; 178 CC0 files from 118 bodies checked, `crates/raw/src/vendor/rw2.rs`), PEF, ORF (uncompressed; colour-filter layout from the file's Exif `CFAPattern`), Hasselblad 3FR with uncompressed raw data (X1D, X1D II, X2D, CFV 100C: a TIFF without a DNG version tag that describes its CFA IFD, levels, crop and white balance with DNG's tags; `RawFormat::CfaTiff`, read by `dng::decode_as`, RGGB when the file states no `CFAPattern`; the lossless-JPEG 3FR and FFF files use scan selection value 8 and stay preview-only), Samsung SRW without compression (NX5, NX10, NX11 and the NX20-generation bodies NX20, NX200, NX210, NX1000 and NX1100 as 12-bit packed samples in either bit order, EX1 and WB2000 as 16-bit words; layout, camera-JPEG framing, levels and white balance from the maker note, `crates/raw/src/vendor/srw.rs`; median ΔE vs the camera JPEG 2.2–7.6 on 11 of the 12 files from 10 bodies checked, 10.7 (5.7 registered) on the backlit WB2000 sample). 12-bit NEF black level: maker note `0x003d` is in 14-bit units (D750, D780, D850, D7500, Z 50 checked). Lossy type 2 NEF curves of version `0x44 0x40` (D780, D6, Z 5, Z 6, Z 7, Z 7_2) span a quarter of the sample range, read from the table's own point spacing. CR3 lossless CRX Bayer and version 0x100/0x200 C-RAW decodes to sensor pixels with timed white balance, black/white levels and file crop (the shot's aspect ratio from the maker note's `AspectInfo`); full sensor samples match the reference on M50/R100/R8 (`crates/raw/tests/cr3_corpus.rs`, `docs/cr3.md`). Unsupported coding variants retain metadata and use the embedded JPEG when available; HEVC preview decoding remains unsupported. Missing: CR3 unverified CRX variants, compressed ORF, compressed Samsung SRW (NX1, NX30, NX300, NX500, NX2000, NX3000, NX3300, NX mini: preview only with the reason), NEFs labelled compressed (34713) but stored uncompressed without a `0x0096` table (Z 6 packed 14-bit, D850 12-bit uncompressed), NEF "lossy after split" (preview only: imported with `preview_only` = the decoder's reason, rendered as a rendered JPEG, flagged by a grid badge and loupe/Edit/Info notices, `previewOnly` in `catalog.query`; Reload clears it once the file decodes), TIFF-based raws whose raw data sits in a private block (Phase One / Leaf IIQ, Canon EOS-1D / 1Ds and Kodak DCS TIFFs: IFD0 is only a reduced RGB image next to a much larger declared picture) are recognised from that structure and import as `preview_only` with the reason, instead of opening as a thumbnail-sized ordinary image (`thumbnail_shell` in `crates/raw/src/lib.rs`), Recognised raws LightCraft cannot decode open as `preview_only` with any usable embedded JPEG, also Canon CRW, Minolta MRW and Sigma X3F (containers recognised by magic, preview found by a structural JPEG scan), Kodak KDC (IFD0 without `ImageWidth`), Leaf MOS and Epson ERF (scan), and any decode failure of a recognised raw, not only an unsupported variant; ones with no JPEG still fail with "without an embedded preview", AVIF decode (HEIC decodes only behind the non-default `heif` feature: `crates/heif`, heic-rs, same policy as PhotoCraft; 8/10-bit, alpha, irot/imir/clap, ICC/EXIF/XMP), palette (indexed-colour) TIFF (refused by the `tiff` crate) |
|
||||
|
||||
| LR-IMP-CAMERA-COVERAGE | Camera coverage, verified per model | P0 | 🟡 | `crates/raw/tests/corpus.rs`, `xtask/src/main.rs` | a supported container is not the same as every camera that writes it decoding correctly: ~55 CC0 corpus files plus 6 exact-sensor CR3 cases (plus ~60 NEFs checked by hand and 178 Panasonic / Leica RW2, RWL and RAW files from 118 bodies, checked against each file's own JPEG) against the >1,000 models Lightroom lists; per-model differences still surface (e.g. the CR2 colour-filter layout differed by model until it was read from the file's own tag, issue #85, checked on 35 CR2 bodies; pre-2017 Sony ARWs opened green until their scrambled white balance / black level were read, issue #148, checked on 16 ARWs from 15 Sony bodies; Canon PowerShot and EOS M CR2s whose `ColorData` is stored as bytes opened green until the bytes were read as 16-bit words, 25 files from 25 bodies checked). Compressed RAF sensor arrays verified on 13 Fujifilm bodies (`docs/raf-compression.md`). Grow the CC0 corpus per model and test each sample decodes with plausible colour. Nikon CropArea and untagged-preview colour-space regression tests; private Z 8 crop verified. |
|
||||
| LR-IMP-CULL-AT-IMPORT | Culling analysis at import | P2 | 🟡 | `cmd:photo.analyze` | run Assisted Culling on the imported photos (they're selected after an import); not automatic |
|
||||
| LR-IMP-MOVE | Move on import [Classic] | P1 | ✅ | `cmd:library.import` (`mode` move), `crates/engine/src/import_move.rs`, `crates/engine/src/tests_import_move.rs`, `crates/ui-egui/src/import.rs` | Transfer ▸ Move with Copy's destination, folders (day / month / one folder / custom template), rename template and example destination; XMP sidecars (both namings) move along; each source is removed only after its destination is written (hard link on the same volume, else copied, synced and compared byte for byte) and its catalog record is saved; failed, duplicate, unchecked files keep their sources, taken names get -1, -2…, never overwritten; files already in the destination/library are added in place; sources that can't be removed (read-only card) are kept and reported (`kept`). Undo removes the photos from the library but leaves the moved files at the destination. No Copy as DNG while moving |
|
||||
@@ -295,7 +296,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins.
|
||||
| LR-PROF-ADOBE | Standard raw looks (own equivalents) | P0 | ✅ | `crates/engine/src/presets.rs` (`PROFILES`), `crates/pipeline/src/profiles.rs` | six own looks: Color, Neutral, Vivid, Landscape, Portrait, Monochrome |
|
||||
| LR-PROF-ADAPTIVE | Adaptive profiles | P2 | ⬜ | | |
|
||||
| LR-PROF-CAMERA | Camera-matching looks | P2 | ⬜ | | |
|
||||
| LR-PROF-CAMERACOLOR | Camera colour calibration (own) | P0 | 🟡 | `crates/raw/src/color.rs`, `crates/raw/src/profile.rs`, `crates/raw/src/gaintable.rs`, `crates/engine/src/camera_preview.rs` | DNG files use the colour matrices and the camera-profile look they carry (`ProfileHueSatMap`, `ProfileLookTable`, `ProfileToneCurve`, read from the file at run time per DNG spec ch. 6 — Lightroom-converted DNGs rendered flat and muted without them, issue #138). A DNG's `ProfileGainTableMap` / `ProfileGainTableMap2` (Apple ProRAW's local tone map; both versions, DNG 1.7 precedence, the file's byte order with a big-endian fallback) is read and written back by DNG export but not rendered, as Lightroom Classic renders ProRAW without it (measured on one CC0 iPhone 12 Pro ProRAW); Sony ARW, Nikon NEF, Panasonic RW2, Fujifilm RAF, Canon CR3 and CR2 and Pentax PEF get a guarded file-local fit to their own embedded JPEG — chromaticity matrix + hue/saturation/value table + tone and chroma curves, relative WB — or, with a bundled or local per-model profile pooled from many photos (`lightcraft-cli calibrate`, ARW, NEF and RAF so far, `crates/engine/src/camera_profiles.rs`), the profile's colour and only their own tone/chroma curves (on 29 held-out ILCE-7M4 photos mean ΔE vs the camera JPEG 3.54 → 3.10), kept with smart previews (see docs/camera-preview-colour.md); ARW: on 10 public raw.pixls.us samples from 8 bodies 7 fits were accepted, mean ΔE vs the camera JPEG 17–26 → 3–10; NEF: 13 of 13 decodable samples (D750, D780, D850, D7500, Z 50 and local D7500 shots) accepted, ΔE 13–46 → 3–8 (issue #150); RW2 / RWL: 140 of 174 public samples accepted, from 93 of the 114 bodies with a preview, median ΔE 5.0 with matching lightness (13 of the 14 corpus files with a preview: ΔE 9.6–27 → 2.8–8.9; most rejections are compacts and kit zooms whose camera JPEG is distortion-corrected); a fit that fails the gates on all pixels is retried away from edges, where a lens-corrected camera JPEG still lines up (issue #232: the six distortion-corrected public ILCE-7RM2 samples ΔE 16.6–16.9 → 3.7–3.8, the corpus GX80 19.1 → 7.9); RAF: bundled X-H2S/X-T4 profiles from 201/545 photos; on 150 withheld files mean ΔE improves slightly over per-file fitting (X-H2S 1.70 → 1.62, X-T4 2.43 → 2.34); 23 of 24 references from 13 bodies accepted, including the dark X-H2S samples (8.74 → 2.25) (`docs/camera-preview-colour.md`); this is not measured camera calibration; other raws and rejected fits use a neutral fallback (camera RGB ≈ linear sRGB, `matrix_is_fallback`) with as-shot white balance, so colours are muted and not accurate. Needs our own per-camera calibration: matrices the files carry themselves (e.g. Olympus `ColorMatrix`), fitting each camera to its own embedded JPEG, then chart shots. Adobe matrices / DCPs are never used. Biggest image-quality gap today |
|
||||
| LR-PROF-CAMERACOLOR | Camera colour calibration (own) | P0 | 🟡 | `crates/raw/src/color.rs`, `crates/raw/src/profile.rs`, `crates/raw/src/gaintable.rs`, `crates/engine/src/camera_preview.rs` | DNG files use the colour matrices and the camera-profile look they carry (`ProfileHueSatMap`, `ProfileLookTable`, `ProfileToneCurve`, read from the file at run time per DNG spec ch. 6 — Lightroom-converted DNGs rendered flat and muted without them, issue #138). A DNG's `ProfileGainTableMap` / `ProfileGainTableMap2` (Apple ProRAW's local tone map; both versions, DNG 1.7 precedence, the file's byte order with a big-endian fallback) is read and written back by DNG export but not rendered, as Lightroom Classic renders ProRAW without it (measured on one CC0 iPhone 12 Pro ProRAW); Sony ARW, Nikon NEF, Panasonic RW2, Fujifilm RAF, Canon CR3 and CR2, Pentax PEF and Samsung SRW get a guarded file-local fit to their own embedded JPEG — chromaticity matrix + hue/saturation/value table + tone and chroma curves, relative WB — or, with a bundled or local per-model profile pooled from many photos (`lightcraft-cli calibrate`, ARW, NEF and RAF so far, `crates/engine/src/camera_profiles.rs`), the profile's colour and only their own tone/chroma curves (on 29 held-out ILCE-7M4 photos mean ΔE vs the camera JPEG 3.54 → 3.10), kept with smart previews (see docs/camera-preview-colour.md); ARW: on 10 public raw.pixls.us samples from 8 bodies 7 fits were accepted, mean ΔE vs the camera JPEG 17–26 → 3–10; NEF: 13 of 13 decodable samples (D750, D780, D850, D7500, Z 50 and local D7500 shots) accepted, ΔE 13–46 → 3–8 (issue #150); RW2 / RWL: 140 of 174 public samples accepted, from 93 of the 114 bodies with a preview, median ΔE 5.0 with matching lightness (13 of the 14 corpus files with a preview: ΔE 9.6–27 → 2.8–8.9; most rejections are compacts and kit zooms whose camera JPEG is distortion-corrected); a fit that fails the gates on all pixels is retried away from edges, where a lens-corrected camera JPEG still lines up (issue #232: the six distortion-corrected public ILCE-7RM2 samples ΔE 16.6–16.9 → 3.7–3.8, the corpus GX80 19.1 → 7.9); RAF: bundled X-H2S/X-T4 profiles from 201/545 photos; on 150 withheld files mean ΔE improves slightly over per-file fitting (X-H2S 1.70 → 1.62, X-T4 2.43 → 2.34); 23 of 24 references from 13 bodies accepted, including the dark X-H2S samples (8.74 → 2.25) (`docs/camera-preview-colour.md`); this is not measured camera calibration; other raws and rejected fits use a neutral fallback (camera RGB ≈ linear sRGB, `matrix_is_fallback`) with as-shot white balance, so colours are muted and not accurate. Needs our own per-camera calibration: matrices the files carry themselves (e.g. Olympus `ColorMatrix`), fitting each camera to its own embedded JPEG, then chart shots. Adobe matrices / DCPs are never used. Biggest image-quality gap today |
|
||||
| LR-PROF-CREATIVE | Creative profiles (own) | P2 | ✅ | `cmd:develop.profile`, `crates/pipeline/src/profiles.rs` | 16 own looks in Film / Cinematic / Muted / B&W (tone + point-curve fades, colour grading, mixer / B&W mix); scale with `ctl:profile.amount`; sliders untouched |
|
||||
| LR-PROF-LEGACY | Legacy profiles | P2 | ⬜ | | |
|
||||
| LR-PROF-NONRAW | Profiles for non-raw files | P0 | ✅ | `cmd:develop.profile` | same looks apply to JPEG/TIFF |
|
||||
|
||||
+5
-1
@@ -37,10 +37,14 @@
|
||||
belonged to another command moves to the new one. Menus show the new keys; agents use `app.setShortcut`.
|
||||
|
||||
### RAW decoding
|
||||
- Samsung SRW files without compression now open as raws: NX5, NX10, NX11, NX20, NX200, NX210, NX1000, NX1100, EX1
|
||||
and WB2000. The compressed ones (NX1, NX30, NX300, NX500, NX2000, NX3000, NX3300, NX mini) still open from their
|
||||
camera JPEG. Photos already imported pick the change up on Reload.
|
||||
- Canon CR2 and Pentax PEF raws get the same starting look fitted to the camera's own JPEG as ARW, NEF, RW2, RAF
|
||||
and CR3, instead of opening flat and desaturated (issue #310). Photos already imported pick it up when re-rendered.
|
||||
- JPEG XL compressed DNGs (DNG 1.7) now open: lossless tiles decode sample for sample (checked on synthetic files);
|
||||
lossy tiles decode too, but no real file has checked them yet. A JPEG XL preview stored in the DNG is used like an
|
||||
lossy tiles decode too, keeping the raw values above 1.0 instead of clipping them (checked on one real file). A tile that cannot be decoded makes the file open from its embedded
|
||||
preview, with the reason, instead of showing a black tile. A JPEG XL preview stored in the DNG is used like an
|
||||
embedded JPEG.
|
||||
- Apple ProRAW's gain table map (its local tone mapping, `ProfileGainTableMap`) is now read and kept when a photo is
|
||||
exported or converted to DNG. It is not applied to the render: Lightroom Classic renders ProRAW without it.
|
||||
|
||||
@@ -10,6 +10,8 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde_json = { workspace = true }
|
||||
# corpus download checksums
|
||||
sha2 = { workspace = true }
|
||||
# precompressed web bundle (`cargo xtask web`); pure Rust
|
||||
flate2 = "1"
|
||||
brotli = { version = "8", default-features = false, features = ["std"] }
|
||||
|
||||
+2
-2
@@ -9,8 +9,8 @@ use std::process::Command;
|
||||
/// Extensions treated as assets (lower-case).
|
||||
const ASSET_EXT: &[&str] = &[
|
||||
"png", "jpg", "jpeg", "gif", "webp", "avif", "svg", "ico", "icns", "bmp", "ttf", "otf", "woff", "woff2", "wav", "mp3", "ogg", "flac", "mp4",
|
||||
"mov", "tif", "tiff", "dng", "cr2", "cr3", "nef", "arw", "raf", "orf", "rw2", "rwl", "raw", "pef", "heic", "jxl", "psd", "cube", "icc", "icm",
|
||||
"xmp",
|
||||
"mov", "tif", "tiff", "dng", "cr2", "cr3", "nef", "arw", "raf", "orf", "rw2", "rwl", "raw", "pef", "srw", "heic", "jxl", "psd", "cube", "icc",
|
||||
"icm", "xmp",
|
||||
];
|
||||
/// Adobe-specific formats we never ship (camera/lens profiles, Lightroom templates).
|
||||
const FORBIDDEN_EXT: &[&str] = &["dcp", "lcp", "lrtemplate", "lrcat", "lrsmcol", "aco", "ase", "abr"];
|
||||
|
||||
+405
-75
@@ -12,7 +12,7 @@ mod stats;
|
||||
mod version;
|
||||
mod web;
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, ExitCode};
|
||||
|
||||
const USAGE: &str = "\
|
||||
@@ -35,7 +35,7 @@ commands:
|
||||
--serve serves it on http://127.0.0.1:<port> (default 8080)
|
||||
ci fmt --check, clippy -D warnings, heif, test, parity refs, layers, assets, wasm (stops at first failure)
|
||||
corpus [--download]
|
||||
show where test corpora live; --download fetches PngSuite and CC0 raw samples (raw.pixls.us) into corpus/
|
||||
show where test corpora live; --download fetches PngSuite and CC0 raw samples (raw.pixls.us) into corpus/ and checks their sha256
|
||||
stats [--exact] count tests and lines per crate (--exact: ask the test harness via `-- --list`)
|
||||
";
|
||||
|
||||
@@ -305,162 +305,451 @@ fn cmd_ci() -> Result<(), String> {
|
||||
}
|
||||
|
||||
/// CC0 raw samples (raw.pixls.us). Small, representative set; extend freely (CC0 only).
|
||||
/// CC0 raw samples from raw.pixls.us (each verified CC0 on the site; files matched by their published SHA-256).
|
||||
/// CC0 raw samples from raw.pixls.us (each verified CC0 on the site).
|
||||
/// The third field is the SHA-256 of the file as downloaded; `cargo xtask corpus --download` checks it.
|
||||
/// One per format / compression variant we decode or deliberately report as unsupported (preview only).
|
||||
const RAW_SAMPLES: &[(&str, &str)] = &[
|
||||
const RAW_SAMPLES: &[(&str, &str, &str)] = &[
|
||||
(
|
||||
"arw-sony-a7m3-compressed.arw",
|
||||
"https://raw.pixls.us/getfile.php/2414/nice/Sony%20-%20ILCE-7M3%20-%2014bit%2014bit%20compressed%20%283:2%29.ARW",
|
||||
"250784580ea527442c09004417bb0eead484f2bf3ee8f9121a776ac65bb50d0f",
|
||||
),
|
||||
(
|
||||
"arw-sony-a7m3-uncompressed.arw",
|
||||
"https://raw.pixls.us/getfile.php/2418/nice/Sony%20-%20ILCE-7M3%20-%2014bit%2014bit%20uncompressed%20%283:2%29.ARW",
|
||||
"ece80551abf64949dbe826985a80f1d9b265401ee4ea6b989e1829549616fdcd",
|
||||
),
|
||||
(
|
||||
"arw-sony-a7m4-14bit.arw",
|
||||
"https://raw.pixls.us/getfile.php/6936/nice/Sony%20-%20ILCE-7M4%20-%2014bit%20%283:2%29.ARW",
|
||||
"639a6d4db881f1359e3ea7e1137b314e59417d24e6d4d862b7202111a2c823b2",
|
||||
),
|
||||
("arw-sony-a7m4-14bit.arw", "https://raw.pixls.us/getfile.php/6936/nice/Sony%20-%20ILCE-7M4%20-%2014bit%20%283:2%29.ARW"),
|
||||
// lossless compressed (Compression 7): L is 2×2 CFA cells per LJ92 sample; M and S are subsampled
|
||||
("arw-sony-a7m4-lossless-l.arw", "https://raw.pixls.us/data/Sony/ILCE-7M4/ILCE-7M4_DSC06674_FullFrame-LossLess-Compressed-Large.ARW"),
|
||||
("arw-sony-a7m4-lossless-m.arw", "https://raw.pixls.us/data/Sony/ILCE-7M4/ILCE-7M4_DSC06675_FullFrame-LossLess-Compressed-Medium.ARW"),
|
||||
("arw-sony-a7m4-lossless-s.arw", "https://raw.pixls.us/data/Sony/ILCE-7M4/ILCE-7M4_DSC06676_FullFrame-LossLess-Compressed-Small.ARW"),
|
||||
(
|
||||
"arw-sony-a7m4-lossless-l.arw",
|
||||
"https://raw.pixls.us/data/Sony/ILCE-7M4/ILCE-7M4_DSC06674_FullFrame-LossLess-Compressed-Large.ARW",
|
||||
"851b43c2116c4139104a5036f83ac3b6a148789b2142214dd7192c13972b25b6",
|
||||
),
|
||||
(
|
||||
"arw-sony-a7m4-lossless-m.arw",
|
||||
"https://raw.pixls.us/data/Sony/ILCE-7M4/ILCE-7M4_DSC06675_FullFrame-LossLess-Compressed-Medium.ARW",
|
||||
"d453005714327addd75bcb99c1c6223173dc92f2a59542d82e6761ef0e0e7571",
|
||||
),
|
||||
(
|
||||
"arw-sony-a7m4-lossless-s.arw",
|
||||
"https://raw.pixls.us/data/Sony/ILCE-7M4/ILCE-7M4_DSC06676_FullFrame-LossLess-Compressed-Small.ARW",
|
||||
"cbbd0930c7d8706dff84c68a2004454266e6fd0d8354f5f76a106b5d776e0223",
|
||||
),
|
||||
// pre-2017 bodies: white balance only in the enciphered maker note, black level only in the SR2SubIFD (#148)
|
||||
("arw-sony-rx100m3.arw", "https://raw.pixls.us/data/Sony/DSC-RX100M3/DSC00734.ARW"),
|
||||
("arw-sony-rx100.arw", "https://raw.pixls.us/data/Sony/DSC-RX100/DSC00838.ARW"),
|
||||
("arw-sony-a7rm2-12bit-uncompressed.arw", "https://raw.pixls.us/data/Sony/ILCE-7RM2/12-bit-uncompressed.ARW"),
|
||||
(
|
||||
"arw-sony-rx100m3.arw",
|
||||
"https://raw.pixls.us/data/Sony/DSC-RX100M3/DSC00734.ARW",
|
||||
"cb81392e3a8810231ce341fb45de3e4d1b9e2f87bc47f3c314d084041ccfd138",
|
||||
),
|
||||
(
|
||||
"arw-sony-rx100.arw",
|
||||
"https://raw.pixls.us/data/Sony/DSC-RX100/DSC00838.ARW",
|
||||
"579a485b5126a25cbd55cbd5dadfa7d09cf021c99cc7d4869f9e56e3f759390b",
|
||||
),
|
||||
(
|
||||
"arw-sony-a7rm2-12bit-uncompressed.arw",
|
||||
"https://raw.pixls.us/data/Sony/ILCE-7RM2/12-bit-uncompressed.ARW",
|
||||
"71e0888396ef52c7e6a990b4e72ebb4d8413a1fc1c35bada449870539cef6e39",
|
||||
),
|
||||
// CR2 colour-filter layouts differ by model (issue #85): CR2CFAPattern 3 (GBRG) and 1 (RGGB) samples
|
||||
("cr2-canon-40d.cr2", "https://raw.pixls.us/data/Canon/EOS%2040D/_MG_0153.CR2"),
|
||||
("cr2-canon-550d.cr2", "https://raw.pixls.us/data/Canon/EOS%20550D/IMG_4047.CR2"),
|
||||
("cr2-canon-5d2.cr2", "https://raw.pixls.us/data/Canon/EOS%205D%20Mark%20II/08.canon.raw.cr2"),
|
||||
("cr2-canon-5dsr.cr2", "https://raw.pixls.us/data/Canon/EOS%205DS%20R/_DSR2002.CR2"),
|
||||
("cr2-canon-6d.cr2", "https://raw.pixls.us/data/Canon/EOS%206D/EOS_6D_RAW.CR2"),
|
||||
("cr2-canon-7d.cr2", "https://raw.pixls.us/data/Canon/EOS%207D/RAW_CANON_EOS_7D-raw.CR2"),
|
||||
("cr2-canon-5d3-sraw2.cr2", "https://raw.pixls.us/getfile.php/773/nice/Canon%20-%20EOS%205D%20Mark%20III%20-%20sRAW2%20%28sRAW%29.CR2"),
|
||||
("cr2-canon-5d3.cr2", "https://raw.pixls.us/getfile.php/771/nice/Canon%20-%20EOS%205D%20Mark%20III.CR2"),
|
||||
("cr2-canon-80d.cr2", "https://raw.pixls.us/getfile.php/1294/nice/Canon%20-%20EOS%2080D%20-%20RAW%20%283:2%29.CR2"),
|
||||
("cr3-canon-m50-craw.cr3", "https://raw.pixls.us/getfile.php/2663/nice/Canon%20-%20EOS%20M50%20-%20CRAW%20%283:2%29.CR3"),
|
||||
("cr3-canon-m50-raw.cr3", "https://raw.pixls.us/getfile.php/4657/nice/Canon%20-%20EOS%20M50%20-%203:2.CR3"),
|
||||
("cr3-canon-r100-raw.cr3", "https://raw.pixls.us/getfile.php/7896/nice/Canon%20-%20EOS%20R100%20-%20RAW%20%283:2%29.CR3"),
|
||||
("cr3-canon-r100-craw.cr3", "https://raw.pixls.us/getfile.php/7897/nice/Canon%20-%20EOS%20R100%20-%20CRAW%20%283:2%29.CR3"),
|
||||
("cr3-canon-r8-raw.cr3", "https://raw.pixls.us/getfile.php/6585/nice/Canon%20-%20EOS%20R8%20-%203:2.CR3"),
|
||||
("cr3-canon-r8-craw.cr3", "https://raw.pixls.us/getfile.php/6587/nice/Canon%20-%20EOS%20R8%20-%203:2.CR3"),
|
||||
(
|
||||
"cr2-canon-40d.cr2",
|
||||
"https://raw.pixls.us/data/Canon/EOS%2040D/_MG_0153.CR2",
|
||||
"775c806358fedddec7622113a5e399a3330bd5a65e35e37ae1108d8bf58067be",
|
||||
),
|
||||
(
|
||||
"cr2-canon-550d.cr2",
|
||||
"https://raw.pixls.us/data/Canon/EOS%20550D/IMG_4047.CR2",
|
||||
"f390e0ba566f0be9af2d8e7f955b815996630c0c3cb13fcfb95d98baec1890f6",
|
||||
),
|
||||
(
|
||||
"cr2-canon-5d2.cr2",
|
||||
"https://raw.pixls.us/data/Canon/EOS%205D%20Mark%20II/08.canon.raw.cr2",
|
||||
"12b3ad6aed9fbcd19ba3bfffcd0769aa5a041d610205eee840bf4b2c420dffae",
|
||||
),
|
||||
(
|
||||
"cr2-canon-5dsr.cr2",
|
||||
"https://raw.pixls.us/data/Canon/EOS%205DS%20R/_DSR2002.CR2",
|
||||
"01b94b0586419d64894ccdf2e7b9f1defeedc7b51dfcd306c622fe784181a9b8",
|
||||
),
|
||||
(
|
||||
"cr2-canon-6d.cr2",
|
||||
"https://raw.pixls.us/data/Canon/EOS%206D/EOS_6D_RAW.CR2",
|
||||
"360842779d08fe4805b4a2fe4979f06fdc6da982c85c3acb181f25bcf4c9f536",
|
||||
),
|
||||
(
|
||||
"cr2-canon-7d.cr2",
|
||||
"https://raw.pixls.us/data/Canon/EOS%207D/RAW_CANON_EOS_7D-raw.CR2",
|
||||
"b5e47c5fcf7332ac03e0134926f17a338a42e68c1fd7f83e16f45f4b767544e8",
|
||||
),
|
||||
(
|
||||
"cr2-canon-5d3-sraw2.cr2",
|
||||
"https://raw.pixls.us/getfile.php/773/nice/Canon%20-%20EOS%205D%20Mark%20III%20-%20sRAW2%20%28sRAW%29.CR2",
|
||||
"a79065e78a6c5bdbc8726e786ade02d54a7d496cc3cb4504d804c924a3d999da",
|
||||
),
|
||||
(
|
||||
"cr2-canon-5d3.cr2",
|
||||
"https://raw.pixls.us/getfile.php/771/nice/Canon%20-%20EOS%205D%20Mark%20III.CR2",
|
||||
"ec069b178b9383d80c72f8cdc1b79bd54ca451e7c20793b109d8ad73bef6bdf6",
|
||||
),
|
||||
(
|
||||
"cr2-canon-80d.cr2",
|
||||
"https://raw.pixls.us/getfile.php/1294/nice/Canon%20-%20EOS%2080D%20-%20RAW%20%283:2%29.CR2",
|
||||
"8f28465cee09844ffad95dbd3ec91923717a329c5ec4f3f405f5b4fce83aaea3",
|
||||
),
|
||||
(
|
||||
"cr3-canon-m50-craw.cr3",
|
||||
"https://raw.pixls.us/getfile.php/2663/nice/Canon%20-%20EOS%20M50%20-%20CRAW%20%283:2%29.CR3",
|
||||
"15384b775867ec4c42b11882837f1e368cedc0561832ffab271221e6bb80be4c",
|
||||
),
|
||||
(
|
||||
"cr3-canon-m50-raw.cr3",
|
||||
"https://raw.pixls.us/getfile.php/4657/nice/Canon%20-%20EOS%20M50%20-%203:2.CR3",
|
||||
"1ba9ad6b51b315b88820eb0fe5fd7f52bc136ba37b1a5ca76825ebfe25b18058",
|
||||
),
|
||||
(
|
||||
"cr3-canon-r100-raw.cr3",
|
||||
"https://raw.pixls.us/getfile.php/7896/nice/Canon%20-%20EOS%20R100%20-%20RAW%20%283:2%29.CR3",
|
||||
"6d83217d58a5e6d2dabcf470430e91a16676b60531c025b81fa7453fc74b0521",
|
||||
),
|
||||
(
|
||||
"cr3-canon-r100-craw.cr3",
|
||||
"https://raw.pixls.us/getfile.php/7897/nice/Canon%20-%20EOS%20R100%20-%20CRAW%20%283:2%29.CR3",
|
||||
"0b66842b2fe00329ebc05fe8ab9357ddbbe1a0a2d2e69a2893102de7fdf5c942",
|
||||
),
|
||||
(
|
||||
"cr3-canon-r8-raw.cr3",
|
||||
"https://raw.pixls.us/getfile.php/6585/nice/Canon%20-%20EOS%20R8%20-%203:2.CR3",
|
||||
"7d5c6dbb11ff7e6ee58715d90a20f5d801e8b103f4711ff6ea329ceafcef254c",
|
||||
),
|
||||
(
|
||||
"cr3-canon-r8-craw.cr3",
|
||||
"https://raw.pixls.us/getfile.php/6587/nice/Canon%20-%20EOS%20R8%20-%203:2.CR3",
|
||||
"df33cf394573645ce03dea1b2e9f0b5cc2b7734e9e3391ee7114151414cf2812",
|
||||
),
|
||||
(
|
||||
"dng-adobe-canon-5d3-linear-lj92.dng",
|
||||
"https://raw.pixls.us/getfile.php/1032/nice/Adobe%20DNG%20Converter%20-%20Canon%20EOS%205D%20Mark%20III%20-%20Lossless%20JPEG%20compression%2C%20rgb%20%283:2%29.DNG",
|
||||
"0608c9f277307b745530e61b589c465540611df22f9a1fe2d01237c5b2b4c13b",
|
||||
),
|
||||
(
|
||||
"dng-adobe-canon-5d3-lj92.dng",
|
||||
"https://raw.pixls.us/getfile.php/1024/nice/Adobe%20DNG%20Converter%20-%20Canon%20EOS%205D%20Mark%20III%20-%2016bit%2016bit%20Lossless%20JPEG%20compression%20%283:2%29.DNG",
|
||||
"3118116735d4f6dd01fd9d6a80ee9aac52a2c2debdc6dcd112bd5e3059044a55",
|
||||
),
|
||||
(
|
||||
"dng-adobe-canon-5d3-lossy.dng",
|
||||
"https://raw.pixls.us/getfile.php/1023/nice/Adobe%20DNG%20Converter%20-%20Canon%20EOS%205D%20Mark%20III%20-%20Lossy%20JPEG%20compression%20%283:2%29.DNG",
|
||||
"159326856c29073e845c3c5a9ecf98c6474f43ca15798a88ad5e2baecd0664b7",
|
||||
),
|
||||
// Apple ProRAW (iPhone 12 Pro, iOS 14.3): LinearRaw LJ92 tiles, ProfileToneCurve, ProfileGainTableMap, a sky matte.
|
||||
// sha256 e91e77a4533ed7cce551d83330676ea5c47dd5e55fb38adda7819366afdbdfc2
|
||||
("dng-apple-iphone12pro-proraw.dng", "https://raw.pixls.us/data/Apple/iPhone%2012%20Pro/IMG_1361.DNG"),
|
||||
(
|
||||
"dng-apple-iphone12pro-proraw.dng",
|
||||
"https://raw.pixls.us/data/Apple/iPhone%2012%20Pro/IMG_1361.DNG",
|
||||
"e91e77a4533ed7cce551d83330676ea5c47dd5e55fb38adda7819366afdbdfc2",
|
||||
),
|
||||
(
|
||||
"dng-canon-5d3-14bit-small.dng",
|
||||
"https://raw.pixls.us/getfile.php/2204/nice/Canon%20-%20EOS%205D%20Mark%20III%20-%2014bit%2014bit%20%282.3471882640587%29.dng",
|
||||
"1d77dcc6cdb839b10c1948f0b4471984ab48cf15e973fb77520584cee72b603e",
|
||||
),
|
||||
(
|
||||
"dng-canon-5d3-16bit-169.dng",
|
||||
"https://raw.pixls.us/getfile.php/2649/nice/Canon%20-%20EOS%205D%20Mark%20III%20-%2016bit%20%2816:9%29.dng",
|
||||
"bc29ac8a37c800346749b9a5a1adc2722357f7f0e49df05a7104454881604b92",
|
||||
),
|
||||
(
|
||||
"dng-canon-5d3-16bit.dng",
|
||||
"https://raw.pixls.us/getfile.php/885/nice/Canon%20-%20EOS%205D%20Mark%20III%20-%2016bit%2016bit%20RAW.dng",
|
||||
"6275226bfa2d86ba479130999d045ea0b8da6be4b011130c9eb91cd6a0eb2233",
|
||||
),
|
||||
(
|
||||
"dng-google-pixel2xl.dng",
|
||||
"https://raw.pixls.us/getfile.php/2206/nice/Google%20-%20Pixel%202%20XL%20-%2016bit%20%284:3%29.dng",
|
||||
"5541093a369f0487fef92bcd1f5fef1be6e4e3cec2aa4d52ad38d2211f3309d7",
|
||||
),
|
||||
(
|
||||
"dng-ricoh-gr3.dng",
|
||||
"https://raw.pixls.us/getfile.php/3115/nice/Ricoh%20-%20GR%20III%20-%2014bit%20%283:2%29.DNG",
|
||||
"05513ee72f7cac4165534c9f64995412084f414a033c80c473c79fd94154292e",
|
||||
),
|
||||
("dng-canon-5d3-16bit-169.dng", "https://raw.pixls.us/getfile.php/2649/nice/Canon%20-%20EOS%205D%20Mark%20III%20-%2016bit%20%2816:9%29.dng"),
|
||||
("dng-canon-5d3-16bit.dng", "https://raw.pixls.us/getfile.php/885/nice/Canon%20-%20EOS%205D%20Mark%20III%20-%2016bit%2016bit%20RAW.dng"),
|
||||
("dng-google-pixel2xl.dng", "https://raw.pixls.us/getfile.php/2206/nice/Google%20-%20Pixel%202%20XL%20-%2016bit%20%284:3%29.dng"),
|
||||
("dng-ricoh-gr3.dng", "https://raw.pixls.us/getfile.php/3115/nice/Ricoh%20-%20GR%20III%20-%2014bit%20%283:2%29.DNG"),
|
||||
(
|
||||
"nef-nikon-d5100-lossless.nef",
|
||||
"https://raw.pixls.us/getfile.php/1597/nice/Nikon%20-%20D5100%20-%2014bit%2014bit%20compressed%20%28Lossless%29%20%283:2%29.nef",
|
||||
"3ce328830942381648be26b372a54b317fdb27b63c1aa9c2ee1fce076a2e1b4f",
|
||||
),
|
||||
(
|
||||
"nef-nikon-d5100-uncompressed.nef",
|
||||
"https://raw.pixls.us/getfile.php/1598/nice/Nikon%20-%20D5100%20-%2014bit%2014bit%20uncompressed%20%283:2%29.nef",
|
||||
"2fb2f1b87a3b89de5ba9e56527f21a912ca624f2fa2a594cef108ee8d7180412",
|
||||
),
|
||||
(
|
||||
"nef-nikon-d7000-lossy12.nef",
|
||||
"https://raw.pixls.us/getfile.php/961/nice/Nikon%20-%20D7000%20-%2012bit%2012bit%20compressed%20%28Lossy%20%28type%202%29%29%20%283:2%29.NEF",
|
||||
"986db2bf9ed08cf095bb94cfbaf02f0e89d123edad28b63c05133e7815749486",
|
||||
),
|
||||
(
|
||||
"nef-nikon-d7500-lossless12.nef",
|
||||
"https://raw.pixls.us/getfile.php/1532/nice/Nikon%20-%20D7500%20-%2012bit%2012bit%20compressed%20%28Lossless%29%20%283:2%29.NEF",
|
||||
"1f4e58a961d3d317fa39a7e1c8948b825651e968c8cd06e49e4239f612faccad",
|
||||
),
|
||||
(
|
||||
"nef-nikon-d7500-lossless14.nef",
|
||||
"https://raw.pixls.us/getfile.php/1534/nice/Nikon%20-%20D7500%20-%2014bit%2014bit%20compressed%20%28Lossless%29%20%283:2%29.NEF",
|
||||
"430b4f1be4e53a011861b63294ad19fdc0353ec3ea62104f77d4c193e3dd3fc8",
|
||||
),
|
||||
(
|
||||
"nrw-nikon-b700-uncompressed.nrw",
|
||||
"https://raw.pixls.us/getfile.php/1621/nice/Nikon%20-%20COOLPIX%20B700%20-%2012bit%2012bit%20uncompressed%20%284:3%29.NRW",
|
||||
"a175d5892304e79282add07eacf4bdc498ca3f050af48cf989c080355b4735ec",
|
||||
),
|
||||
(
|
||||
"orf-olympus-e1.orf",
|
||||
"https://raw.pixls.us/getfile.php/1800/nice/Olympus%20-%20E-1%20-%2016bit%20%284:3%29.ORF",
|
||||
"042286653fbae5b085bef4a4e626145385ea6e82e817f166b4904abcef64457c",
|
||||
),
|
||||
(
|
||||
"orf-olympus-e400.orf",
|
||||
"https://raw.pixls.us/getfile.php/2151/nice/Olympus%20-%20E-400%20-%2016bit%20%284:3%29.ORF",
|
||||
"1354e0ac98ea90820227650031d8215637a2aaabd15de995c5a1e00b1986261b",
|
||||
),
|
||||
(
|
||||
"orf-olympus-em1.orf",
|
||||
"https://raw.pixls.us/getfile.php/1051/nice/Olympus%20-%20E-M1%20-%2016bit%20%284:3%29.orf",
|
||||
"19ba17e778ae802d4039f0e88bb1cd68825780788b8058c479067e32b9a63eda",
|
||||
),
|
||||
(
|
||||
"orf-olympus-em10iii.orf",
|
||||
"https://raw.pixls.us/getfile.php/1787/nice/Olympus%20-%20E-M10%20Mark%20III%20-%2016bit%20%284:3%29.ORF",
|
||||
"11d90cbb564ad2c660c7a8d5d81d99c2c018aa27fdec45ed6c461fa5d24f942b",
|
||||
),
|
||||
(
|
||||
"orf-olympus-xz2.orf",
|
||||
"https://raw.pixls.us/getfile.php/1432/nice/Olympus%20-%20XZ-2%20-%2012bit%20%284:3%29.orf",
|
||||
"675fefe7c9e99281783b987889d723b5f46692f5c20e19fed5e422c7c6d2636b",
|
||||
),
|
||||
(
|
||||
"pef-pentax-k10d.pef",
|
||||
"https://raw.pixls.us/getfile.php/2239/nice/Pentax%20-%20K10D%20-%2012bit%2012bit%20compressed%20%283:2%29.PEF",
|
||||
"e35ae4154a468be3154f5f462e884ba5941f010d3e8f23d347fbec14809f44d3",
|
||||
),
|
||||
(
|
||||
"pef-pentax-k3.pef",
|
||||
"https://raw.pixls.us/getfile.php/1075/nice/Pentax%20-%20K-3%20-%2014bit%20%283:2%29.PEF",
|
||||
"abfb3907b53734b5f353e8de0c6173470c7aad07f9cecc10f723d222a49dec9a",
|
||||
),
|
||||
(
|
||||
"pef-pentax-k5iis.pef",
|
||||
"https://raw.pixls.us/getfile.php/1198/nice/Pentax%20-%20K-5%20II%20s%20-%2014bit%20%283:2%29.PEF",
|
||||
"e579e7360c35e512f9c3a5c0eb9519575b87b489c6f86be82590f7706a7ed17b",
|
||||
),
|
||||
// Samsung SRW, one file per packing: 12-bit MSB-first (NX10), 12-bit LSB-first (NX20), 16-bit words (EX1).
|
||||
(
|
||||
"srw-samsung-nx10.srw",
|
||||
"https://raw.pixls.us/getfile.php/5913/nice/Samsung%20-%20NX10%20-%2012bit%20%283:2%29.SRW",
|
||||
"dead7b4ad83b739ff7b8a1218882589f66cb14efc09f79eeba7a1dcbf7abd29f",
|
||||
),
|
||||
(
|
||||
"srw-samsung-nx20.srw",
|
||||
"https://raw.pixls.us/getfile.php/5370/nice/Samsung%20-%20NX20%20-%2012bit%20%283:2%29.SRW",
|
||||
"8a367db506b8f7a6661a6f33a730467cfb3bfd7e46020a9f0df702330a7b45bb",
|
||||
),
|
||||
(
|
||||
"srw-samsung-ex1.srw",
|
||||
"https://raw.pixls.us/getfile.php/1204/nice/Samsung%20-%20EX1%20-%2014bit%20%284:3%29.SRW",
|
||||
"a084890fd9d7995ceb746c18ecf5be48c95baf14286d5c0c480cc3daa25f3e37",
|
||||
),
|
||||
("orf-olympus-e1.orf", "https://raw.pixls.us/getfile.php/1800/nice/Olympus%20-%20E-1%20-%2016bit%20%284:3%29.ORF"),
|
||||
("orf-olympus-e400.orf", "https://raw.pixls.us/getfile.php/2151/nice/Olympus%20-%20E-400%20-%2016bit%20%284:3%29.ORF"),
|
||||
("orf-olympus-em1.orf", "https://raw.pixls.us/getfile.php/1051/nice/Olympus%20-%20E-M1%20-%2016bit%20%284:3%29.orf"),
|
||||
("orf-olympus-em10iii.orf", "https://raw.pixls.us/getfile.php/1787/nice/Olympus%20-%20E-M10%20Mark%20III%20-%2016bit%20%284:3%29.ORF"),
|
||||
("orf-olympus-xz2.orf", "https://raw.pixls.us/getfile.php/1432/nice/Olympus%20-%20XZ-2%20-%2012bit%20%284:3%29.orf"),
|
||||
("pef-pentax-k10d.pef", "https://raw.pixls.us/getfile.php/2239/nice/Pentax%20-%20K10D%20-%2012bit%2012bit%20compressed%20%283:2%29.PEF"),
|
||||
("pef-pentax-k3.pef", "https://raw.pixls.us/getfile.php/1075/nice/Pentax%20-%20K-3%20-%2014bit%20%283:2%29.PEF"),
|
||||
("pef-pentax-k5iis.pef", "https://raw.pixls.us/getfile.php/1198/nice/Pentax%20-%20K-5%20II%20s%20-%2014bit%20%283:2%29.PEF"),
|
||||
(
|
||||
"raf-fuji-xa2-12bit-bayer.raf",
|
||||
"https://raw.pixls.us/getfile.php/2883/nice/Fujifilm%20-%20X-A2%20-%2012bit%2012bit%20uncompressed%20%283:2%29.RAF",
|
||||
"58ad687b23138c3c49ebeb6c1f80daf13185349be364b947e2bf27e39a462d2f",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xa5-14bit-bayer.raf",
|
||||
"https://raw.pixls.us/getfile.php/2526/nice/Fujifilm%20-%20X-A5%20-%2014bit%2014bit%20uncompressed%20%283:2%29.RAF",
|
||||
"46daaf8c2b07f40bd01f737181d048a497bb0a87615a1fde65809fdd5de86bf1",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xe1-12bit.raf",
|
||||
"https://raw.pixls.us/getfile.php/3098/nice/Fujifilm%20-%20X-E1%20-%2012bit%2012bit%20uncompressed%20%283:2%29.RAF",
|
||||
"0fa01fa1a674bcfa40f659225b4e66c5bedf15d2f78407ca1966dc8f38cd0e9f",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xt20-14bit.raf",
|
||||
"https://raw.pixls.us/getfile.php/1177/nice/Fujifilm%20-%20X-T20%20-%2014bit%2014bit%20uncompressed%20%283:2%29.RAF",
|
||||
"afe552f4a2794c7aa2376dd826165354b243dcf6f2296b01505dded8c8e4be76",
|
||||
),
|
||||
("raf-fuji-xe1-12bit.raf", "https://raw.pixls.us/getfile.php/3098/nice/Fujifilm%20-%20X-E1%20-%2012bit%2012bit%20uncompressed%20%283:2%29.RAF"),
|
||||
("raf-fuji-xt20-14bit.raf", "https://raw.pixls.us/getfile.php/1177/nice/Fujifilm%20-%20X-T20%20-%2014bit%2014bit%20uncompressed%20%283:2%29.RAF"),
|
||||
(
|
||||
"raf-fuji-xt20-compressed.raf",
|
||||
"https://raw.pixls.us/getfile.php/1178/nice/Fujifilm%20-%20X-T20%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF",
|
||||
"a23045101b2912e64f154100d646c228c0d16585d21e95e0d1a006e537f7784c",
|
||||
),
|
||||
(
|
||||
"raw-panasonic-fz50.raw",
|
||||
"https://raw.pixls.us/getfile.php/2234/nice/Panasonic%20-%20DMC-FZ50%20-%204:3.RAW",
|
||||
"a776a941758214bf4d1342e2b89ed756bfcad5877e038cf9df3bdda502c432ea",
|
||||
),
|
||||
(
|
||||
"raw-panasonic-fz8.raw",
|
||||
"https://raw.pixls.us/getfile.php/2282/nice/Panasonic%20-%20DMC-FZ8%20-%204:3.RAW",
|
||||
"a11f1534f3092a4d5f2b57d182f6b06f8d26cf766eb463c3e02b26a338b63b8b",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-fz1000m2-4x3.rw2",
|
||||
"https://raw.pixls.us/getfile.php/4706/nice/Panasonic%20-%20DC-FZ10002%20-%204:3.RW2",
|
||||
"ab30922774bd829a992471d89eeefe1e829565a886d1477958130a2fa4fb77a1",
|
||||
),
|
||||
("raw-panasonic-fz50.raw", "https://raw.pixls.us/getfile.php/2234/nice/Panasonic%20-%20DMC-FZ50%20-%204:3.RAW"),
|
||||
("raw-panasonic-fz8.raw", "https://raw.pixls.us/getfile.php/2282/nice/Panasonic%20-%20DMC-FZ8%20-%204:3.RAW"),
|
||||
("rw2-panasonic-fz1000m2-4x3.rw2", "https://raw.pixls.us/getfile.php/4706/nice/Panasonic%20-%20DC-FZ10002%20-%204:3.RW2"),
|
||||
// Fujifilm predictive compression: older X-Trans, 40 MP X-Trans, 14/16-bit GFX; both compression modes.
|
||||
(
|
||||
"raf-fuji-gfx100-3773.raf",
|
||||
"https://raw.pixls.us/getfile.php/3773/nice/Fujifilm%20-%20GFX%20100%20-%2016bit%2016bit%20compressed%20%284:3%29.RAF",
|
||||
"018781e126842596fcf3876b3fb6cd8973dc3936ccc753ee8648618be8889287",
|
||||
),
|
||||
(
|
||||
"raf-fuji-gfx100-3775.raf",
|
||||
"https://raw.pixls.us/getfile.php/3775/nice/Fujifilm%20-%20GFX%20100%20-%2014bit%2014bit%20compressed%20%284:3%29.RAF",
|
||||
"0e7dad9819a748bb3d3be26bbf5770d4968e2d80b9cb41d97036bf7c27f7bfe7",
|
||||
),
|
||||
(
|
||||
"raf-fuji-gfx100rf-8091.raf",
|
||||
"https://raw.pixls.us/getfile.php/8091/nice/Fujifilm%20-%20GFX100RF%20-%2016bit%20compressed%20%284:3%29.RAF",
|
||||
"14bed5ac8191d6f2a7c82984e3bf6470ad4740df329293eac42b8018cc6a668a",
|
||||
),
|
||||
("raf-fuji-gfx100rf-8091.raf", "https://raw.pixls.us/getfile.php/8091/nice/Fujifilm%20-%20GFX100RF%20-%2016bit%20compressed%20%284:3%29.RAF"),
|
||||
(
|
||||
"raf-fuji-gfx100s-4495.raf",
|
||||
"https://raw.pixls.us/getfile.php/4495/nice/Fujifilm%20-%20GFX100S%20-%2016bit%2016bit%20lossless%20compressed%20%284:3%29.RAF",
|
||||
"a7bf8f061b603f7ef5fd29276eb6b2ac57355ca2f589a340117bfff0d03c065c",
|
||||
),
|
||||
(
|
||||
"raf-fuji-gfx100s-4503.raf",
|
||||
"https://raw.pixls.us/getfile.php/4503/nice/Fujifilm%20-%20GFX100S%20-%2016bit%2016bit%20compressed%20%284:3%29.RAF",
|
||||
"1bb4608b61861a3d9e85b0ca6c00689b8ba4620cbe8579374d958c21ed08edf0",
|
||||
),
|
||||
(
|
||||
"raf-fuji-gfx50s-1435.raf",
|
||||
"https://raw.pixls.us/getfile.php/1435/nice/Fujifilm%20-%20GFX%2050S%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF",
|
||||
"a7124d13910e0241a6b035154bc1fcdb7be4439371d043b1e1950a0251feb51e",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xe5-8509.raf",
|
||||
"https://raw.pixls.us/getfile.php/8509/nice/Fujifilm%20-%20X-E5%20-%2014bit%20lossy%20compressed%20%283:2%29.RAF",
|
||||
"28f56cc9784164209e92fae209bdcc78267bd692aa2fb964595485f7e93692ab",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xh2-6001.raf",
|
||||
"https://raw.pixls.us/getfile.php/6001/nice/Fujifilm%20-%20X-H2%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF",
|
||||
"caeadb2666f6c5d9bbe32fbe692c2079b6a7e028f8514a13173f360656d08507",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xh2-6002.raf",
|
||||
"https://raw.pixls.us/getfile.php/6002/nice/Fujifilm%20-%20X-H2%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF",
|
||||
"109be38f59963c9b8ff289ebe1f5ec9168c52f609e5b601432bd194520b4605b",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xm5-7748.raf",
|
||||
"https://raw.pixls.us/getfile.php/7748/nice/Fujifilm%20-%20X-M5%20-%2014bit%20compressed%20%283:2%29.RAF",
|
||||
"c76ce546e15727abb4ba201471010c554403176963a1a6b8cc017462606429e5",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xt2-865.raf",
|
||||
"https://raw.pixls.us/getfile.php/865/nice/Fujifilm%20-%20X-T2%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF",
|
||||
"f756ad5be551a8b3e1b503a8a6f7e8e637fba519c0b0f8c79dd58e94b0287602",
|
||||
),
|
||||
("raf-fuji-xe5-8509.raf", "https://raw.pixls.us/getfile.php/8509/nice/Fujifilm%20-%20X-E5%20-%2014bit%20lossy%20compressed%20%283:2%29.RAF"),
|
||||
("raf-fuji-xh2-6001.raf", "https://raw.pixls.us/getfile.php/6001/nice/Fujifilm%20-%20X-H2%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF"),
|
||||
("raf-fuji-xh2-6002.raf", "https://raw.pixls.us/getfile.php/6002/nice/Fujifilm%20-%20X-H2%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF"),
|
||||
("raf-fuji-xm5-7748.raf", "https://raw.pixls.us/getfile.php/7748/nice/Fujifilm%20-%20X-M5%20-%2014bit%20compressed%20%283:2%29.RAF"),
|
||||
("raf-fuji-xt2-865.raf", "https://raw.pixls.us/getfile.php/865/nice/Fujifilm%20-%20X-T2%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF"),
|
||||
(
|
||||
"raf-fuji-xt4-3914.raf",
|
||||
"https://raw.pixls.us/getfile.php/3914/nice/Fujifilm%20-%20X-T4%20-%2014bit%2014bit%20lossless%20compressed%20%283:2%29.RAF",
|
||||
"07ea642656074834ffe5c661870a612ffa2e80fc1218a05166ee966bbe5f6fae",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xt4-3918.raf",
|
||||
"https://raw.pixls.us/getfile.php/3918/nice/Fujifilm%20-%20X-T4%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF",
|
||||
"871836938ffa695f4c41765ddb107c2703ce755f7f3ced7e10f9e04746f1734e",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xt5-6122.raf",
|
||||
"https://raw.pixls.us/getfile.php/6122/nice/Fujifilm%20-%20X-T5%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF",
|
||||
"b2a067e7fda16338f15c2532caa12b238b2415dc1c75a96a6d24d3445cfd0ae3",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xt5-6123.raf",
|
||||
"https://raw.pixls.us/getfile.php/6123/nice/Fujifilm%20-%20X-T5%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF",
|
||||
"cddd7ae0c43f9280876e5fdcbdf8878718972ebd3d034d8affc8cfc6752d33dc",
|
||||
),
|
||||
(
|
||||
"raf-fuji-xt50-7807.raf",
|
||||
"https://raw.pixls.us/getfile.php/7807/nice/Fujifilm%20-%20X-T50%20-%2014bit%20compressed%20%283:2%29.RAF",
|
||||
"3a8a9ac6d92274969aaa1f55ea669191b98bb8f588d00ec8ec17afd8b3963dea",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-g9-b.rw2",
|
||||
"https://raw.pixls.us/getfile.php/2348/nice/Panasonic%20-%20DC-G9%20-%204:3.RW2",
|
||||
"16282357ce0143ff7d36737f82074662db7aa0790ecd309196f406baf0f586a7",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-g9.rw2",
|
||||
"https://raw.pixls.us/getfile.php/2585/nice/Panasonic%20-%20DC-G9%20-%204:3.RW2",
|
||||
"7a56babeb19f9bde3eaa532a4c8d1b11637a0f550963a1182ec845722c65583d",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-gh1.rw2",
|
||||
"https://raw.pixls.us/getfile.php/1323/nice/Panasonic%20-%20DMC-GH1%20-%204:3.RW2",
|
||||
"2b846f932c665f196af9af97b18469730331b60b2c62811969be7db245ce6cce",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-gh5.rw2",
|
||||
"https://raw.pixls.us/getfile.php/1517/nice/Panasonic%20-%20DC-GH5%20-%204:3.RW2",
|
||||
"dae2281df393d8c3c1b8c513d00985f1834ae0e69ce18fbd571370c25d95d22b",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-gh5m2.rw2",
|
||||
"https://raw.pixls.us/getfile.php/5082/nice/Panasonic%20-%20DC-GH5M2%20-%204:3.RW2",
|
||||
"0ea0efd2a9442d8c68508e4196e29f1da4577407dc3df5add4cf360f70493c7b",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-gh5s.rw2",
|
||||
"https://raw.pixls.us/getfile.php/2603/nice/Panasonic%20-%20DC-GH5S%20-%204:3.RW2",
|
||||
"b617151876e62b492641cef14b8c98ee3797e5014d24c6af1c780715a501fa53",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-gh6.rw2",
|
||||
"https://raw.pixls.us/getfile.php/5876/nice/Panasonic%20-%20DC-GH6%20-%204:3.RW2",
|
||||
"07996bca904a178d02c5f56ee03a8dce775a0e1150c1beeaec51d72ff5e700d0",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-gx80.rw2",
|
||||
"https://raw.pixls.us/getfile.php/1569/nice/Panasonic%20-%20DMC-GX80%20-%204:3.RW2",
|
||||
"6e9a419c70c2124630912b2934027c635d3871e3d22dd91fed0655d58cfb5382",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-s1.rw2",
|
||||
"https://raw.pixls.us/getfile.php/3038/nice/Panasonic%20-%20DC-S1%20-%203:2.RW2",
|
||||
"45309a096d6010281e2fa310e6ec2fe1bcc02947fa37b1a2414cfc7377ce4ac6",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-s5-format7.rw2",
|
||||
"https://raw.pixls.us/getfile.php/6339/nice/Panasonic%20-%20DC-S5%20-%203:2.RW2",
|
||||
"3615c85207097180b231f0c89d018eb3f1a71988780612921eb0a05246bca706",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-s5m2.rw2",
|
||||
"https://raw.pixls.us/getfile.php/7790/nice/Panasonic%20-%20DC-S5M2%20-%2014bit%20%283:2%29.RW2",
|
||||
"53a171521a7f9af6910b4ab61c5a4a42de0cceb24166e15b07eee1e00675028f",
|
||||
),
|
||||
(
|
||||
"rw2-panasonic-s9.rw2",
|
||||
"https://raw.pixls.us/getfile.php/7702/nice/Panasonic%20-%20DC-S9%20-%203:2.RW2",
|
||||
"290ad5b4e396cb241a59459b2613d2b0659a03ff654e5d5368b3376d35812e50",
|
||||
),
|
||||
(
|
||||
"rwl-leica-dlux7.rwl",
|
||||
"https://raw.pixls.us/getfile.php/4204/nice/Leica%20-%20D-Lux%207%20-%204:3.RWL",
|
||||
"4f6db6492da83f7f2a5012591e370557249d1ed44d4c948784016605b799327c",
|
||||
),
|
||||
("raf-fuji-xt4-3918.raf", "https://raw.pixls.us/getfile.php/3918/nice/Fujifilm%20-%20X-T4%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF"),
|
||||
("raf-fuji-xt5-6122.raf", "https://raw.pixls.us/getfile.php/6122/nice/Fujifilm%20-%20X-T5%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF"),
|
||||
("raf-fuji-xt5-6123.raf", "https://raw.pixls.us/getfile.php/6123/nice/Fujifilm%20-%20X-T5%20-%2014bit%2014bit%20compressed%20%283:2%29.RAF"),
|
||||
("raf-fuji-xt50-7807.raf", "https://raw.pixls.us/getfile.php/7807/nice/Fujifilm%20-%20X-T50%20-%2014bit%20compressed%20%283:2%29.RAF"),
|
||||
("rw2-panasonic-g9-b.rw2", "https://raw.pixls.us/getfile.php/2348/nice/Panasonic%20-%20DC-G9%20-%204:3.RW2"),
|
||||
("rw2-panasonic-g9.rw2", "https://raw.pixls.us/getfile.php/2585/nice/Panasonic%20-%20DC-G9%20-%204:3.RW2"),
|
||||
("rw2-panasonic-gh1.rw2", "https://raw.pixls.us/getfile.php/1323/nice/Panasonic%20-%20DMC-GH1%20-%204:3.RW2"),
|
||||
("rw2-panasonic-gh5.rw2", "https://raw.pixls.us/getfile.php/1517/nice/Panasonic%20-%20DC-GH5%20-%204:3.RW2"),
|
||||
("rw2-panasonic-gh5m2.rw2", "https://raw.pixls.us/getfile.php/5082/nice/Panasonic%20-%20DC-GH5M2%20-%204:3.RW2"),
|
||||
("rw2-panasonic-gh5s.rw2", "https://raw.pixls.us/getfile.php/2603/nice/Panasonic%20-%20DC-GH5S%20-%204:3.RW2"),
|
||||
("rw2-panasonic-gh6.rw2", "https://raw.pixls.us/getfile.php/5876/nice/Panasonic%20-%20DC-GH6%20-%204:3.RW2"),
|
||||
("rw2-panasonic-gx80.rw2", "https://raw.pixls.us/getfile.php/1569/nice/Panasonic%20-%20DMC-GX80%20-%204:3.RW2"),
|
||||
("rw2-panasonic-s1.rw2", "https://raw.pixls.us/getfile.php/3038/nice/Panasonic%20-%20DC-S1%20-%203:2.RW2"),
|
||||
("rw2-panasonic-s5-format7.rw2", "https://raw.pixls.us/getfile.php/6339/nice/Panasonic%20-%20DC-S5%20-%203:2.RW2"),
|
||||
("rw2-panasonic-s5m2.rw2", "https://raw.pixls.us/getfile.php/7790/nice/Panasonic%20-%20DC-S5M2%20-%2014bit%20%283:2%29.RW2"),
|
||||
("rw2-panasonic-s9.rw2", "https://raw.pixls.us/getfile.php/7702/nice/Panasonic%20-%20DC-S9%20-%203:2.RW2"),
|
||||
("rwl-leica-dlux7.rwl", "https://raw.pixls.us/getfile.php/4204/nice/Leica%20-%20D-Lux%207%20-%204:3.RWL"),
|
||||
];
|
||||
|
||||
fn cmd_corpus(download: bool) -> Result<(), String> {
|
||||
@@ -479,16 +768,41 @@ Tests that use a corpus skip cleanly when it is absent.
|
||||
}
|
||||
let dest = corpus.join("raw");
|
||||
std::fs::create_dir_all(&dest).map_err(|e| format!("create {}: {e}", dest.display()))?;
|
||||
for (name, url) in RAW_SAMPLES {
|
||||
let mut bad = Vec::new();
|
||||
for (name, url, sha256) in RAW_SAMPLES {
|
||||
let out = dest.join(name);
|
||||
if out.exists() {
|
||||
continue;
|
||||
if !out.exists() {
|
||||
let mut curl = Command::new("curl");
|
||||
curl.args(["-fsSL", "-o"]).arg(&out).arg(url);
|
||||
run(curl, &format!("curl {url}"))?;
|
||||
}
|
||||
let actual = file_sha256(&out)?;
|
||||
if actual != *sha256 {
|
||||
// delete it, so the next run downloads it again instead of trusting it
|
||||
std::fs::remove_file(&out).map_err(|e| format!("remove {}: {e}", out.display()))?;
|
||||
bad.push(format!("{name}: sha256 is {actual}, expected {sha256} (file deleted; source {url})"));
|
||||
}
|
||||
let mut curl = Command::new("curl");
|
||||
curl.args(["-fsSL", "-o"]).arg(&out).arg(url);
|
||||
run(curl, &format!("curl {url}"))?;
|
||||
}
|
||||
Ok(())
|
||||
if bad.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
Err(format!(
|
||||
"{} corpus file(s) do not match their pinned sha256:
|
||||
{}",
|
||||
bad.len(),
|
||||
bad.join(
|
||||
"
|
||||
"
|
||||
)
|
||||
))
|
||||
}
|
||||
|
||||
fn file_sha256(path: &Path) -> Result<String, String> {
|
||||
use sha2::{Digest, Sha256};
|
||||
let mut file = std::fs::File::open(path).map_err(|e| format!("open {}: {e}", path.display()))?;
|
||||
let mut hash = Sha256::new();
|
||||
std::io::copy(&mut file, &mut hash).map_err(|e| format!("read {}: {e}", path.display()))?;
|
||||
Ok(format!("{:x}", hash.finalize()))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -507,3 +821,19 @@ mod ci_jobs_tests {
|
||||
assert_eq!(ci_jobs(None, 2), 2);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod corpus_tests {
|
||||
use super::RAW_SAMPLES;
|
||||
|
||||
#[test]
|
||||
fn every_sample_has_a_unique_name_and_a_sha256() {
|
||||
let mut names: Vec<_> = RAW_SAMPLES.iter().map(|s| s.0).collect();
|
||||
names.sort_unstable();
|
||||
names.dedup();
|
||||
assert_eq!(names.len(), RAW_SAMPLES.len(), "duplicate file name");
|
||||
for (name, _, sha256) in RAW_SAMPLES {
|
||||
assert!(sha256.len() == 64 && sha256.bytes().all(|b| matches!(b, b'0'..=b'9' | b'a'..=b'f')), "{name}: not a lowercase sha256");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user