//! Streaming SHA-1 of a file's bytes, for matching originals with remote services that identify //! assets by it (Immich stores the SHA-1 of every original). SHA-1 is used here as a content //! identifier agreed with those services, not for security. //! //! Made to ride along an existing read pass: wrap the reader the import already uses in a //! [`HashingReader`] (or feed the chunks it reads to a [`Sha1`]) and take the digest at the end, //! so a file is read from disk once. [`sha1_reader`] and [`sha1_file`] cover a standalone pass //! (back-filling old catalogs). //! //! Pure computation; builds everywhere, wasm32 included. #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] use std::io::Read; use std::path::Path; use sha1::Digest; /// A SHA-1 digest. #[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] pub struct Sha1Digest(pub [u8; 20]); impl Sha1Digest { /// Lower-case hex (40 characters). pub fn to_hex(&self) -> String { self.0.iter().map(|b| format!("{b:02x}")).collect() } /// Parse 40 hex characters (any case). pub fn from_hex(s: &str) -> Option { let s = s.trim(); if s.len() != 40 { return None; } let mut out = [0u8; 20]; for (i, b) in out.iter_mut().enumerate() { *b = u8::from_str_radix(s.get(i * 2..i * 2 + 2)?, 16).ok()?; } Some(Sha1Digest(out)) } /// Standard base64, the form Immich's API uses for `checksum`. pub fn to_base64(&self) -> String { const A: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; let mut out = String::with_capacity(28); for chunk in self.0.chunks(3) { let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)]; let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); for i in 0..4 { if i <= chunk.len() { out.push(char::from(A.get(((n >> (18 - 6 * i)) & 63) as usize).copied().unwrap_or(b'='))); } else { out.push('='); } } } out } } impl std::fmt::Display for Sha1Digest { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(&self.to_hex()) } } impl std::fmt::Debug for Sha1Digest { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "Sha1Digest({})", self.to_hex()) } } /// An incremental SHA-1. #[derive(Clone, Default)] pub struct Sha1(sha1::Sha1); impl Sha1 { pub fn new() -> Sha1 { Sha1::default() } pub fn update(&mut self, bytes: &[u8]) { self.0.update(bytes); } pub fn finish(self) -> Sha1Digest { Sha1Digest(self.0.finalize().into()) } } /// Hashes everything read through it. pub struct HashingReader { inner: R, hasher: Sha1, bytes: u64, } impl HashingReader { pub fn new(inner: R) -> Self { HashingReader { inner, hasher: Sha1::new(), bytes: 0 } } /// Bytes read so far. pub fn bytes_read(&self) -> u64 { self.bytes } /// The digest of everything read so far, and the reader back. Read to the end first for the /// whole file's SHA-1. pub fn finish(self) -> (R, Sha1Digest) { (self.inner, self.hasher.finish()) } } impl Read for HashingReader { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { let n = self.inner.read(buf)?; self.hasher.update(buf.get(..n).unwrap_or_default()); self.bytes = self.bytes.saturating_add(n as u64); Ok(n) } } /// SHA-1 of everything `r` yields. pub fn sha1_reader(r: impl Read) -> std::io::Result { let mut h = HashingReader::new(r); std::io::copy(&mut h, &mut std::io::sink())?; Ok(h.finish().1) } /// SHA-1 of a file's bytes. pub fn sha1_file(path: &Path) -> std::io::Result { sha1_reader(std::io::BufReader::with_capacity(1 << 20, std::fs::File::open(path)?)) } #[cfg(test)] mod tests { use super::*; #[test] fn known_vectors() { assert_eq!(sha1_reader(&b""[..]).unwrap().to_hex(), "da39a3ee5e6b4b0d3255bfef95601890afd80709"); let abc = sha1_reader(&b"abc"[..]).unwrap(); assert_eq!(abc.to_hex(), "a9993e364706816aba3e25717850c26c9cd0d89d"); assert_eq!(abc.to_base64(), "qZk+NkcGgWq6PiVxeFDCbJzQ2J0="); assert_eq!(Sha1Digest::from_hex(&abc.to_hex().to_uppercase()), Some(abc)); assert_eq!(Sha1Digest::from_hex("xyz"), None); assert_eq!(Sha1Digest::from_hex(&"é".repeat(20)), None); } #[test] fn hashing_reader_rides_along() { let data: Vec = (0..100_000u32).map(|i| i as u8).collect(); let mut r = HashingReader::new(&data[..]); let mut copy = Vec::new(); r.read_to_end(&mut copy).unwrap(); assert_eq!(r.bytes_read(), 100_000); let (_, d) = r.finish(); let mut h = Sha1::new(); for c in data.chunks(777) { h.update(c); } assert_eq!(h.finish(), d); assert_eq!(copy, data); } }