diff --git a/crates/hash/Cargo.toml b/crates/hash/Cargo.toml new file mode 100644 index 0000000000..fffc4ec9d4 --- /dev/null +++ b/crates/hash/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "dac-hash" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Streaming content hashes (L0): SHA-1 of originals, as remote services such as Immich store them" + +[dependencies] +sha1 = { version = "0.10", default-features = false } + +[lints] +workspace = true diff --git a/crates/hash/src/lib.rs b/crates/hash/src/lib.rs new file mode 100644 index 0000000000..e57856d377 --- /dev/null +++ b/crates/hash/src/lib.rs @@ -0,0 +1,167 @@ +//! Streaming SHA-1 of a file's bytes, for matching originals with remote services that identify +//! assets by it (Immich stores the SHA-1 of every original). SHA-1 is used here as a content +//! identifier agreed with those services, not for security. +//! +//! Made to ride along an existing read pass: wrap the reader the import already uses in a +//! [`HashingReader`] (or feed the chunks it reads to a [`Sha1`]) and take the digest at the end, +//! so a file is read from disk once. [`sha1_reader`] and [`sha1_file`] cover a standalone pass +//! (back-filling old catalogs). +//! +//! Pure computation; builds everywhere, wasm32 included. + +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::io::Read; +use std::path::Path; + +use sha1::Digest; + +/// A SHA-1 digest. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Sha1Digest(pub [u8; 20]); + +impl Sha1Digest { + /// Lower-case hex (40 characters). + pub fn to_hex(&self) -> String { + self.0.iter().map(|b| format!("{b:02x}")).collect() + } + + /// Parse 40 hex characters (any case). + pub fn from_hex(s: &str) -> Option { + let s = s.trim(); + if s.len() != 40 { + return None; + } + let mut out = [0u8; 20]; + for (i, b) in out.iter_mut().enumerate() { + *b = u8::from_str_radix(s.get(i * 2..i * 2 + 2)?, 16).ok()?; + } + Some(Sha1Digest(out)) + } + + /// Standard base64, the form Immich's API uses for `checksum`. + pub fn to_base64(&self) -> String { + const A: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(28); + for chunk in self.0.chunks(3) { + let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)]; + let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); + for i in 0..4 { + if i <= chunk.len() { + out.push(char::from(A.get(((n >> (18 - 6 * i)) & 63) as usize).copied().unwrap_or(b'='))); + } else { + out.push('='); + } + } + } + out + } +} + +impl std::fmt::Display for Sha1Digest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.to_hex()) + } +} + +impl std::fmt::Debug for Sha1Digest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Sha1Digest({})", self.to_hex()) + } +} + +/// An incremental SHA-1. +#[derive(Clone, Default)] +pub struct Sha1(sha1::Sha1); + +impl Sha1 { + pub fn new() -> Sha1 { + Sha1::default() + } + + pub fn update(&mut self, bytes: &[u8]) { + self.0.update(bytes); + } + + pub fn finish(self) -> Sha1Digest { + Sha1Digest(self.0.finalize().into()) + } +} + +/// Hashes everything read through it. +pub struct HashingReader { + inner: R, + hasher: Sha1, + bytes: u64, +} + +impl HashingReader { + pub fn new(inner: R) -> Self { + HashingReader { inner, hasher: Sha1::new(), bytes: 0 } + } + + /// Bytes read so far. + pub fn bytes_read(&self) -> u64 { + self.bytes + } + + /// The digest of everything read so far, and the reader back. Read to the end first for the + /// whole file's SHA-1. + pub fn finish(self) -> (R, Sha1Digest) { + (self.inner, self.hasher.finish()) + } +} + +impl Read for HashingReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read(buf)?; + self.hasher.update(buf.get(..n).unwrap_or_default()); + self.bytes = self.bytes.saturating_add(n as u64); + Ok(n) + } +} + +/// SHA-1 of everything `r` yields. +pub fn sha1_reader(r: impl Read) -> std::io::Result { + let mut h = HashingReader::new(r); + std::io::copy(&mut h, &mut std::io::sink())?; + Ok(h.finish().1) +} + +/// SHA-1 of a file's bytes. +pub fn sha1_file(path: &Path) -> std::io::Result { + sha1_reader(std::io::BufReader::with_capacity(1 << 20, std::fs::File::open(path)?)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn known_vectors() { + assert_eq!(sha1_reader(&b""[..]).unwrap().to_hex(), "da39a3ee5e6b4b0d3255bfef95601890afd80709"); + let abc = sha1_reader(&b"abc"[..]).unwrap(); + assert_eq!(abc.to_hex(), "a9993e364706816aba3e25717850c26c9cd0d89d"); + assert_eq!(abc.to_base64(), "qZk+NkcGgWq6PiVxeFDCbJzQ2J0="); + assert_eq!(Sha1Digest::from_hex(&abc.to_hex().to_uppercase()), Some(abc)); + assert_eq!(Sha1Digest::from_hex("xyz"), None); + assert_eq!(Sha1Digest::from_hex(&"é".repeat(20)), None); + } + + #[test] + fn hashing_reader_rides_along() { + let data: Vec = (0..100_000u32).map(|i| i as u8).collect(); + let mut r = HashingReader::new(&data[..]); + let mut copy = Vec::new(); + r.read_to_end(&mut copy).unwrap(); + assert_eq!(r.bytes_read(), 100_000); + let (_, d) = r.finish(); + let mut h = Sha1::new(); + for c in data.chunks(777) { + h.update(c); + } + assert_eq!(h.finish(), d); + assert_eq!(copy, data); + } +}