LR-IMP-CAMERA-COVERAGE: Sony DSC-R1 reads its SR2SubIFD (black, white balance, crop): seed derived from the key

crates/engine/src/media.rs

@@ -62,7 +62,7 @@ impl SettingsHashes {
 }
 
 /// Bump when the pipeline's output changes, to invalidate cached thumbnails.
-pub const RENDER_CACHE_VERSION: u64 = 37;
+pub const RENDER_CACHE_VERSION: u64 = 44;
 
 /// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).
 pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];

crates/raw/src/vendor/arw.rs

@@ -29,8 +29,9 @@
 //! ([`word16_order`]). Measured on the CC0 DSC-R1 (raw.pixls.us 3221): StripByteCounts 20780544 = 3984×2608×2; read
 //! big-endian the maximum is 16368 and the image is smooth (neighbour roughness 0.012 against 0.249), read
 //! little-endian the maximum is 65340 (noise). Black stays the default 512 (the masked columns read 511.4) and white
-//! comes from the data. The crop and white balance are NOT addressed here: the file has no maker-note `FullImageSize`
-//! and no plain WB tag (they live in its `SR2SubIFD`), so it opens uncropped with unit multipliers.
+//! comes from the data. The file has no maker-note `FullImageSize` and no plain WB or crop tag; they live in its
+//! `SR2SubIFD` (key `63 be 01 3e`, see [`super::sr2`]): black `0x7300`, as-shot WB `0x7303` (G, R, B, G) and the crop
+//! `0x74c3` (x0, y0, x1, y1), each read only when no other source exists.
 //!
 //! Also: uncompressed 16-bit ARW, and lossless-compressed ARW (Compression 7, ILCE-7M4 and later): LJ92 tiles whose
 //! frames hold one 2×2 CFA cell per four-component sample ([`read_quad_tiles`]). Other lossless-JPEG layouts go
@@ -49,6 +50,11 @@ const BLACK_LEVEL: u16 = 0x7310;
 const WB_RGGB: u16 = 0x7313;
 const CROP_TOP_LEFT: u16 = 0x74c7;
 const CROP_SIZE: u16 = 0x74c8;
+/// `SR2SubIFD` tags of the oldest layout (DSC-R1): per-channel black, as-shot WB in G, R, B, G order, and the crop
+/// `(x0, y0, x1, y1)`. Used only when nothing else gives the value.
+const SR2_OLD_BLACK: u16 = 0x7300;
+const SR2_OLD_WB: u16 = 0x7303;
+const SR2_OLD_CROP: u16 = 0x74c3;
 const YCBCR_COEFFICIENTS: u16 = 529;
 const REFERENCE_BLACK_WHITE: u16 = 532;
 /// Maker-note tags (ExifTool Sony tag names): the enciphered `Tag2010` block and `FullImageSize` (height, width).
@@ -431,12 +437,27 @@ fn read_ycbcr_tiles(bytes: &[u8], info: &ImageInfo, raw: &Ifd, mode: Mode) -> Re
 /// on the DSLR-A450/A500/A550 (27152 bytes: 354–365) and the DSLR-A700 (62112 bytes: 975), whose black is 512 by
 /// ExifTool and by the data floor (issue #535).
 fn sr2_black(sr2: &super::sr2::SubIfd) -> Option<f32> {
-    match sr2.short_bytes(BLACK_LEVEL)? {
+    match sr2.short_bytes(BLACK_LEVEL).or_else(|| sr2.short_bytes(SR2_OLD_BLACK))? {
         (3, plain) if plain.len() == 8 => sr2_black_levels(plain, sr2.order()),
         _ => None,
     }
 }
 
+/// The DSC-R1's as-shot gains `[R/G, 1, B/G]` from `0x7303` (G, R, B, G), when plausible.
+fn sr2_old_wb(sr2: &super::sr2::SubIfd) -> Option<[f32; 3]> {
+    let v = sr2.shorts(SR2_OLD_WB).filter(|v| v.len() == 4 && v.iter().all(|&x| x > 0.0))?;
+    let g = (v[0] + v[3]) / 2.0;
+    let (r, b) = ((v[1] / g) as f32, (v[2] / g) as f32);
+    ((0.25..=16.0).contains(&r) && (0.25..=16.0).contains(&b)).then_some([r, 1.0, b])
+}
+
+/// The DSC-R1's crop from `0x74c3` (x0, y0, x1, y1): an even origin (CFA phase) and a window inside the `w × h` frame.
+fn sr2_old_crop(sr2: &super::sr2::SubIfd, w: usize, h: usize) -> Option<Rect> {
+    let v = sr2.shorts(SR2_OLD_CROP).filter(|v| v.len() == 4)?;
+    let [x0, y0, x1, y1] = [v[0] as usize, v[1] as usize, v[2] as usize, v[3] as usize];
+    (x0 % 2 == 0 && y0 % 2 == 0 && x1 > x0 && y1 > y0 && x1 <= w && y1 <= h).then(|| Rect::new(x0, y0, x1 - x0, y1 - y0))
+}
+
 /// One black level from the four deciphered per-channel levels (their mean), when they are near-equal and plausible.
 fn sr2_black_levels(plain: &[u8], order: lightcraft_tiff::ByteOrder) -> Option<f32> {
     let levels: Vec<u16> = (0..4).filter_map(|i| order.read_u16(plain, 2 * i)).collect();
@@ -606,12 +627,15 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
         .f64s(WB_RGGB)
         .and_then(|v| rggb_gains(&v))
         .or_else(|| rggb_gains(&sr2()?.shorts(WB_RGGB)?))
-        .or_else(|| mn.as_ref().and_then(|m| tag2010_wb(&model, m.ifd.bytes(MN_TAG2010)?, m.order)));
+        .or_else(|| mn.as_ref().and_then(|m| tag2010_wb(&model, m.ifd.bytes(MN_TAG2010)?, m.order)))
+        .or_else(|| sr2_old_wb(sr2()?));
     let crop = match (raw.u64s(CROP_TOP_LEFT).as_deref(), raw.u64s(CROP_SIZE).as_deref()) {
         (Some([x, y]), Some([cw, ch])) if *cw > 0 && *ch > 0 => Rect::new(*x as usize, *y as usize, *cw as usize, *ch as usize).clipped(w, h),
         _ => {
             let exif_size = tiff.exif().and_then(|e| Some((e.u64(t::PIXEL_X_DIMENSION)?, e.u64(t::PIXEL_Y_DIMENSION)?)));
-            default_crop(raw, mn.as_ref(), exif_size, w, h)
+            let crop = default_crop(raw, mn.as_ref(), exif_size, w, h);
+            // no crop from any other source: the old SR2SubIFD's
+            if crop == Rect::new(0, 0, w, h) { sr2().and_then(|s| sr2_old_crop(s, w, h)).unwrap_or(crop) } else { crop }
         }
     };
     let mut metadata = lightcraft_meta::from_tiff(&tiff);
@@ -1004,6 +1028,29 @@ mod tests {
         file
     }
 
+    #[test]
+    fn dsc_r1_sr2_subifd_gives_black_wb_and_crop() {
+        use lightcraft_tiff::ByteOrder::Little;
+        let key = [0x63, 0xbe, 0x01, 0x3e];
+        let enc = |crop: [u16; 4], wb: [u16; 4]| {
+            let b = super::super::sr2::tests::block_keyed(
+                230812,
+                &[(SR2_OLD_BLACK, 3, &[514, 511, 512, 513]), (SR2_OLD_WB, 3, &wb), (SR2_OLD_CROP, 3, &crop)],
+                Little,
+                key,
+            );
+            super::super::sr2::SubIfd::decrypt_keyed(&b, 230812, Little, key)
+        };
+        let sub = enc([16, 8, 3904, 2600], [256, 500, 385, 256]);
+        assert_eq!(sr2_black(&sub), Some(512.5));
+        assert_eq!(sr2_old_wb(&sub), Some([500.0 / 256.0, 1.0, 385.0 / 256.0]));
+        assert_eq!(sr2_old_crop(&sub, 3984, 2608), Some(Rect::new(16, 8, 3888, 2592)));
+        // implausible values are dropped, not guessed
+        assert_eq!(sr2_old_crop(&enc([15, 8, 3904, 2600], [0, 1, 1, 0]), 3984, 2608), None);
+        assert_eq!(sr2_old_crop(&sub, 3900, 2608), None);
+        assert_eq!(sr2_old_wb(&enc([16, 8, 3904, 2600], [0, 1, 1, 0])), None);
+    }
+
     #[test]
     fn white_balance_prefers_the_applied_levels() {
         const KEY: [u8; 4] = [0x11, 0x22, 0x33, 0x44];

crates/raw/src/vendor/sr2.rs

@@ -33,9 +33,17 @@
 //!    (8 bytes at position 2510, issue #148, from the plain black levels of the DSC-RX100M6 and ILCE-7M3) and the
 //!    positional tables the black level was read with until issue #535; the unit tests pin all of them.
 //!
-//! Only that key is supported. The one other key seen, in a DSC-R1 SR2 file, gives a different keystream that obeys
-//! the same recurrence from its own first 127 words, but one sample per key cannot show how a seed follows from a
-//! key, so files with another key get `None`.
+//!
+//! **Keys other than `11 22 33 44`: the seed derivation is a recalled scheme (generic memory, no code copied),
+//! confirmed because it reproduces [`SEED`] — the keystream #575 measured from 126 CC0 files — exactly, and because
+//! it turns the DSC-R1's block (another key) into a valid directory.** [`seed_from_key`] takes the key as the
+//! little-endian `u32` of tag `0x7221`, fills 128 state words from the linear congruential generator
+//! `k ← k·48828125 + 1` (four words, the fourth then shifted left one bit with the top bit of `w0 ^ w2` shifted in),
+//! extends them with `w[i] = (w[i−4] ⊕ w[i−2]) << 1 | (w[i−3] ⊕ w[i−1]) >> 31` to word 126, and the first 127 words of
+//! `w[i & 127] = w[(i+1) & 127] ⊕ w[(i+65) & 127]` from `i = 127` are the seed of the recurrence above. For key
+//! `11 22 33 44` that is [`SEED`], word for word (unit test `seed_derivation_reproduces_the_measured_seed`; chance
+//! of an accident 2⁻⁴⁰⁶⁴). For the DSC-R1's key `63 be 01 3e` the 11932-byte block becomes a well-formed 101-entry
+//! directory (black `0x7300`, white balance `0x7303`, crop `0x74c3`).
 
 use lightcraft_tiff::{ByteOrder, Ifd, ParseOptions, tags as t};
 
@@ -67,10 +75,42 @@ const SEED: [u32; 127] = [
     0xcdc6a2de, 0x26e6b330, 0x6c073d46, 0x058e2f27, 0x43833f30, 0x46d1382e, 0x5f0804ec,
 ];
 
-/// XOR `block` with the keystream from its first byte on (whole 4-byte words; a shorter tail stays as it is).
-/// Encrypts and decrypts alike.
+/// The seed (first 127 keystream words) for `key`, the little-endian `u32` of tag `0x7221`; see the module docs.
+fn seed_from_key(key: u32) -> [u32; 127] {
+    let mut p = [0u32; 128];
+    let mut k = key;
+    for w in p.iter_mut().take(4) {
+        k = k.wrapping_mul(48_828_125).wrapping_add(1);
+        *w = k;
+    }
+    p[3] = (p[3] << 1) | ((p[0] ^ p[2]) >> 31);
+    for i in 4..127 {
+        p[i] = ((p[i - 4] ^ p[i - 2]) << 1) | ((p[i - 3] ^ p[i - 1]) >> 31);
+    }
+    let mut seed = [0u32; 127];
+    for (j, out) in seed.iter_mut().enumerate() {
+        let i = 127 + j;
+        p[i & 127] = p[(i + 1) & 127] ^ p[(i + 65) & 127];
+        *out = p[i & 127];
+    }
+    seed
+}
+
+/// The seed for the four key bytes stored in the file: the measured [`SEED`] for [`KEY`], else the derived one.
+fn seed_for(key: [u8; 4]) -> [u32; 127] {
+    if key == KEY { SEED } else { seed_from_key(u32::from_le_bytes(key)) }
+}
+
+/// XOR `block` with the keystream of [`KEY`] from its first byte on (whole 4-byte words; a shorter tail stays as
+/// it is). Encrypts and decrypts alike.
+#[cfg(test)]
 fn apply_keystream(block: &mut [u8]) {
-    let mut ring = SEED;
+    apply_keystream_with(block, SEED);
+}
+
+/// [`apply_keystream`] for the keystream starting with `seed`.
+fn apply_keystream_with(block: &mut [u8], seed: [u32; 127]) {
+    let mut ring = seed;
     for (n, word) in block.as_chunks_mut::<4>().0.iter_mut().enumerate() {
         // `ring[i]` holds k[n − 127] until it is replaced by k[n]; k[n − 63] sits at (n − 63) mod 127
         let i = n % SEED.len();
@@ -93,26 +133,30 @@ pub(crate) struct SubIfd {
 
 impl SubIfd {
     /// Find and decrypt the file's `SR2SubIFD`; `None` when it has none, it lies outside the file, is larger than
-    /// [`MAX_LEN`] or uses another key than [`KEY`].
+    /// [`MAX_LEN`].
     pub(crate) fn read(bytes: &[u8], ifd0: &Ifd, order: ByteOrder) -> Option<Self> {
         let at = order.read_u32(ifd0.bytes(t::DNG_PRIVATE_DATA)?, 0)?;
         let opts = ParseOptions { max_ifds: 1, max_depth: 0, follow_children: false, ..Default::default() };
         let (private, _) = lightcraft_tiff::parse_ifd_at(bytes, u64::from(at), order, 0, false, &opts).ok()?;
-        if private.bytes(SUBIFD_KEY)? != KEY {
-            return None;
-        }
+        let key: [u8; 4] = private.bytes(SUBIFD_KEY)?.try_into().ok()?;
         let start = usize::try_from(private.u64(SUBIFD_OFFSET)?).ok()?;
         let len = usize::try_from(private.u64(SUBIFD_LENGTH)?).ok()?;
         if len > MAX_LEN {
             return None;
         }
-        Some(Self::decrypt(bytes.get(start..start.checked_add(len)?)?, start, order))
+        Some(Self::decrypt_keyed(bytes.get(start..start.checked_add(len)?)?, start, order, key))
     }
 
     /// Decrypt `block`, the encrypted directory found at file offset `start`.
+    #[cfg(test)]
     pub(crate) fn decrypt(block: &[u8], start: usize, order: ByteOrder) -> Self {
+        Self::decrypt_keyed(block, start, order, KEY)
+    }
+
+    /// [`SubIfd::decrypt`] for a block stored with `key` (tag `0x7221`).
+    pub(crate) fn decrypt_keyed(block: &[u8], start: usize, order: ByteOrder, key: [u8; 4]) -> Self {
         let mut plain = block.to_vec();
-        apply_keystream(&mut plain);
+        apply_keystream_with(&mut plain, seed_for(key));
         Self { start, plain, order }
     }
 
@@ -182,6 +226,14 @@ pub(crate) mod tests {
         plain
     }
 
+    /// [`block`] encrypted with another `key`.
+    pub(crate) fn block_keyed(start: usize, entries: &[(u16, u16, &[u16])], order: ByteOrder, key: [u8; 4]) -> Vec<u8> {
+        let mut b = block(start, entries, order);
+        apply_keystream(&mut b); // undo the default key
+        apply_keystream_with(&mut b, seed_for(key));
+        b
+    }
+
     /// Encrypt (or decrypt) `block` in place, as a file stores an `SR2SubIFD` at its first byte.
     pub(crate) fn encrypt(block: &mut [u8]) {
         apply_keystream(block);
@@ -236,6 +288,32 @@ pub(crate) mod tests {
         }
     }
 
+    /// The proof of the recalled scheme: derived from key `11 22 33 44`, the seed is exactly the keystream #575
+    /// measured from the CC0 files.
+    #[test]
+    fn seed_derivation_reproduces_the_measured_seed() {
+        assert_eq!(seed_from_key(u32::from_le_bytes(KEY)), SEED);
+        assert_ne!(seed_from_key(u32::from_le_bytes(KEY)), seed_from_key(u32::from_le_bytes(KEY) ^ 1));
+        assert_eq!(seed_for(KEY), SEED);
+    }
+
+    #[test]
+    fn another_key_decrypts_a_block() {
+        // the DSC-R1's key and layout: black 0x7300, WB 0x7303 (G, R, B, G), crop 0x74c3
+        let key = [0x63, 0xbe, 0x01, 0x3e];
+        let entries: [(u16, u16, &[u16]); 3] =
+            [(0x7300, SHORT, &[514, 511, 512, 513]), (0x7303, SHORT, &[256, 500, 385, 256]), (0x74c3, SHORT, &[16, 8, 3904, 2600])];
+        for order in [Little, Big] {
+            let enc = block_keyed(230812, &entries, order, key);
+            let sub = SubIfd::decrypt_keyed(&enc, 230812, order, key);
+            assert_eq!(sub.shorts(0x7300), Some(vec![514.0, 511.0, 512.0, 513.0]), "{order:?}");
+            assert_eq!(sub.shorts(0x7303), Some(vec![256.0, 500.0, 385.0, 256.0]));
+            assert_eq!(sub.shorts(0x74c3), Some(vec![16.0, 8.0, 3904.0, 2600.0]));
+            // the wrong key (or the default one) gives noise
+            assert_eq!(SubIfd::decrypt(&enc, 230812, order).shorts(0x7300), None);
+        }
+    }
+
     #[test]
     fn keystream_follows_its_recurrence() {
         let mut zeros = vec![0u8; 62114];