diff --git a/Cargo.lock b/Cargo.lock index 680a9a24a7..ce695f589f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -310,6 +310,18 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "argon2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072" +dependencies = [ + "base64ct", + "blake2", + "cpufeatures", + "password-hash", +] + [[package]] name = "arrayvec" version = "0.7.8" @@ -641,6 +653,15 @@ dependencies = [ "no_std_io2", ] +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest", +] + [[package]] name = "block" version = "0.1.6" @@ -1268,6 +1289,21 @@ dependencies = [ "serde", ] +[[package]] +name = "dac-credentials" +version = "0.4.0" +dependencies = [ + "argon2", + "chacha20poly1305", + "dac-brand", + "getrandom 0.3.4", + "log", + "serde", + "serde_json", + "zbus", + "zeroize", +] + [[package]] name = "dac-denoise" version = "0.4.0" @@ -1472,6 +1508,13 @@ dependencies = [ "wgpu", ] +[[package]] +name = "dac-hash" +version = "0.4.0" +dependencies = [ + "sha1", +] + [[package]] name = "dac-heif" version = "0.4.0" @@ -1522,6 +1565,21 @@ dependencies = [ "serde_json", ] +[[package]] +name = "dac-net" +version = "0.4.0" +dependencies = [ + "dac-brand", + "flate2", + "log", + "rustls", + "rustls-rustcrypto", + "serde", + "serde_json", + "sha2", + "webpki-roots", +] + [[package]] name = "dac-pipeline" version = "0.4.0" @@ -4353,6 +4411,17 @@ dependencies = [ "windows-link", ] +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "paste" version = "1.0.15" @@ -5400,6 +5469,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sha1" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" diff --git a/Cargo.toml b/Cargo.toml index ae671a1eb3..70c966c383 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,9 @@ dac-engine-export = { path = "crates/engine-export" } dac-ui-egui = { path = "crates/ui-egui" } dac-mcp = { path = "crates/mcp" } dac-fetch = { path = "crates/fetch" } +dac-net = { path = "crates/net" } +dac-credentials = { path = "crates/credentials" } +dac-hash = { path = "crates/hash" } dac-heif = { path = "crates/heif" } dac-segment = { path = "crates/segment" } diff --git a/crates/credentials/Cargo.toml b/crates/credentials/Cargo.toml new file mode 100644 index 0000000000..30ffac1ddd --- /dev/null +++ b/crates/credentials/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "dac-credentials" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Secret storage (L0): the OS keychain where reachable in pure Rust, else a passphrase-encrypted file" + +[dependencies] +dac-brand = { workspace = true } + +[lints] +workspace = true + +# Native only: on wasm32 the crate is empty (the web build stores no credentials). +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +zeroize = "1" +getrandom = "0.3" +argon2 = { version = "0.5", default-features = false, features = ["alloc"] } +chacha20poly1305 = { version = "0.10", default-features = false, features = ["alloc"] } + +# Linux and the BSDs: the freedesktop Secret Service over D-Bus, in pure Rust. +[target.'cfg(all(unix, not(target_os = "macos"), not(target_os = "ios"), not(target_os = "android")))'.dependencies] +zbus = { version = "5", default-features = false, features = ["blocking-api", "async-io"] } diff --git a/crates/credentials/src/file.rs b/crates/credentials/src/file.rs new file mode 100644 index 0000000000..c8c12e232d --- /dev/null +++ b/crates/credentials/src/file.rs @@ -0,0 +1,231 @@ +//! The fallback store: one JSON file whose payload is the secrets map encrypted with +//! XChaCha20-Poly1305 under a key derived from a passphrase with Argon2id (RFC 9106). +//! +//! Sources: RFC 9106 (Argon2), RFC 8439 and draft-irtf-cfrg-xchacha (XChaCha20-Poly1305); +//! file layout is own design. + +use std::collections::BTreeMap; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, PoisonError}; + +use chacha20poly1305::aead::{Aead, KeyInit, Payload}; +use chacha20poly1305::{XChaCha20Poly1305, XNonce}; +use serde::{Deserialize, Serialize}; +use zeroize::Zeroizing; + +use crate::{CredError, Key, Secret, SecretStore}; + +const FORMAT: u32 = 1; +/// Bound to the ciphertext so a payload can't be moved into another kind of file. +const AAD: &[u8] = b"credentials-file-v1"; +/// Largest file read. +const MAX_FILE: u64 = 16 * 1024 * 1024; + +/// Argon2id cost for new files (OWASP's minimum recommendation: 19 MiB, 2 passes). +const M_KIB: u32 = 19 * 1024; +const T: u32 = 2; +const P: u32 = 1; + +#[derive(Serialize, Deserialize)] +struct Envelope { + format: u32, + kdf: String, + m_kib: u32, + t: u32, + p: u32, + salt: String, + nonce: String, + data: String, +} + +fn hex(b: &[u8]) -> String { + b.iter().map(|b| format!("{b:02x}")).collect() +} + +fn unhex(s: &str) -> Option> { + if !s.len().is_multiple_of(2) { + return None; + } + (0..s.len() / 2).map(|i| u8::from_str_radix(s.get(i * 2..i * 2 + 2)?, 16).ok()).collect() +} + +fn random() -> Result<[u8; N], CredError> { + let mut b = [0u8; N]; + getrandom::fill(&mut b).map_err(|e| CredError::Io(format!("no randomness: {e}")))?; + Ok(b) +} + +fn derive(passphrase: &Secret, salt: &[u8], m_kib: u32, t: u32, p: u32) -> Result, CredError> { + // bound the cost read from the file: a doctored header must not exhaust memory or time + if !(8..=1024 * 1024).contains(&m_kib) || !(1..=16).contains(&t) || !(1..=16).contains(&p) { + return Err(CredError::Corrupt("key derivation parameters out of range".into())); + } + let params = argon2::Params::new(m_kib, t, p, Some(32)).map_err(|e| CredError::Corrupt(e.to_string()))?; + let argon = argon2::Argon2::new(argon2::Algorithm::Argon2id, argon2::Version::V0x13, params); + let mut key = Zeroizing::new([0u8; 32]); + argon.hash_password_into(passphrase.expose().as_bytes(), salt, key.as_mut()).map_err(|e| CredError::Corrupt(e.to_string()))?; + Ok(key) +} + +type Map = BTreeMap; + +fn map_key(k: &Key) -> String { + format!("{}\n{}", k.service, k.account) +} + +/// Secrets in a passphrase-encrypted file. Every call reads and (for changes) rewrites the file, +/// so several handles on one file stay consistent. +pub struct FileStore { + path: PathBuf, + key: Zeroizing<[u8; 32]>, + salt: Vec, + lock: Mutex<()>, +} + +impl std::fmt::Debug for FileStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("FileStore").field("path", &self.path).finish_non_exhaustive() + } +} + +impl FileStore { + /// Open `path` with `passphrase`, or create an empty store there (written on the first + /// [`SecretStore::set`]). A wrong passphrase for an existing file is [`CredError::WrongPassphrase`]. + pub fn open(path: &Path, passphrase: &Secret) -> Result { + match read_envelope(path)? { + Some(env) => { + let salt = unhex(&env.salt).ok_or_else(|| CredError::Corrupt("bad salt".into()))?; + let key = derive(passphrase, &salt, env.m_kib, env.t, env.p)?; + let store = FileStore { path: path.to_path_buf(), key, salt, lock: Mutex::new(()) }; + store.decrypt(&env)?; // proves the passphrase + Ok(store) + } + None => { + let salt = random::<16>()?.to_vec(); + let key = derive(passphrase, &salt, M_KIB, T, P)?; + Ok(FileStore { path: path.to_path_buf(), key, salt, lock: Mutex::new(()) }) + } + } + } + + pub fn path(&self) -> &Path { + &self.path + } + + fn cipher(&self) -> Result { + XChaCha20Poly1305::new_from_slice(self.key.as_ref()).map_err(|_| CredError::Corrupt("bad key length".into())) + } + + fn decrypt(&self, env: &Envelope) -> Result { + if env.format != FORMAT || env.kdf != "argon2id" { + return Err(CredError::Corrupt(format!("unknown format {}", env.format))); + } + let nonce: [u8; 24] = unhex(&env.nonce).and_then(|n| n.try_into().ok()).ok_or_else(|| CredError::Corrupt("bad nonce".into()))?; + let data = unhex(&env.data).ok_or_else(|| CredError::Corrupt("bad payload".into()))?; + let plain = + Zeroizing::new(self.cipher()?.decrypt(&XNonce::from(nonce), Payload { msg: &data, aad: AAD }).map_err(|_| CredError::WrongPassphrase)?); + let map: Map = serde_json::from_slice(&plain).map_err(|_| CredError::Corrupt("bad payload".into()))?; + Ok(map) + } + + fn load(&self) -> Result { + match read_envelope(&self.path)? { + Some(env) => { + if unhex(&env.salt).as_deref() != Some(&self.salt[..]) { + return Err(CredError::Corrupt("the file was replaced while open; open it again".into())); + } + self.decrypt(&env) + } + None => Ok(Map::new()), + } + } + + fn save(&self, map: &Map) -> Result<(), CredError> { + let plain = Zeroizing::new(serde_json::to_vec(map).map_err(|e| CredError::Io(e.to_string()))?); + let nonce = random::<24>()?; + let data = + self.cipher()?.encrypt(&XNonce::from(nonce), Payload { msg: &plain, aad: AAD }).map_err(|_| CredError::Io("encryption failed".into()))?; + let env = Envelope { + format: FORMAT, + kdf: "argon2id".into(), + m_kib: M_KIB, + t: T, + p: P, + salt: hex(&self.salt), + nonce: hex(&nonce), + data: hex(&data), + }; + let text = serde_json::to_vec_pretty(&env).map_err(|e| CredError::Io(e.to_string()))?; + write_atomic(&self.path, &text) + } +} + +fn io(path: &Path, e: std::io::Error) -> CredError { + CredError::Io(format!("{}: {e}", path.display())) +} + +fn read_envelope(path: &Path) -> Result, CredError> { + let meta = match std::fs::metadata(path) { + Ok(m) => m, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(io(path, e)), + }; + if meta.len() > MAX_FILE { + return Err(CredError::Corrupt("file too large".into())); + } + let bytes = std::fs::read(path).map_err(|e| io(path, e))?; + serde_json::from_slice(&bytes).map(Some).map_err(|_| CredError::Corrupt("not a credentials file".into())) +} + +/// Write to a temporary file (owner-only on Unix), sync, then rename over `path`. +fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), CredError> { + if let Some(dir) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + std::fs::create_dir_all(dir).map_err(|e| io(dir, e))?; + } + let mut name = path.file_name().map(|n| n.to_os_string()).unwrap_or_default(); + name.push(".tmp"); + let tmp = path.with_file_name(name); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let mut f = opts.open(&tmp).map_err(|e| io(&tmp, e))?; + f.write_all(bytes).and_then(|_| f.sync_all()).map_err(|e| io(&tmp, e))?; + drop(f); + std::fs::rename(&tmp, path).map_err(|e| io(path, e)) +} + +impl SecretStore for FileStore { + fn name(&self) -> &'static str { + "encrypted file" + } + + fn get(&self, key: &Key) -> Result, CredError> { + let _g = self.lock.lock().unwrap_or_else(PoisonError::into_inner); + Ok(self.load()?.get(&map_key(key)).map(|s| Secret::new(s.clone()))) + } + + fn set(&self, key: &Key, secret: &Secret) -> Result<(), CredError> { + let _g = self.lock.lock().unwrap_or_else(PoisonError::into_inner); + let mut map = self.load()?; + map.insert(map_key(key), secret.expose().to_string()); + self.save(&map)?; + log::info!("credentials: stored a secret for {} in the encrypted file", key.service); + Ok(()) + } + + fn delete(&self, key: &Key) -> Result { + let _g = self.lock.lock().unwrap_or_else(PoisonError::into_inner); + let mut map = self.load()?; + let removed = map.remove(&map_key(key)).is_some(); + if removed { + self.save(&map)?; + log::info!("credentials: removed a secret for {} from the encrypted file", key.service); + } + Ok(removed) + } +} diff --git a/crates/credentials/src/lib.rs b/crates/credentials/src/lib.rs new file mode 100644 index 0000000000..13e6c22178 --- /dev/null +++ b/crates/credentials/src/lib.rs @@ -0,0 +1,118 @@ +//! Where account secrets (API keys, tokens, passwords) live: never in the catalog, settings, +//! logs, MCP output or crash reports. +//! +//! - [`os_store`]: the platform keychain, where it is reachable in pure Rust without `unsafe`. +//! Linux and the BSDs use the freedesktop Secret Service (GNOME Keyring, KWallet) over D-Bus +//! with `zbus`. macOS Keychain and Windows Credential Manager need FFI, which this workspace +//! keeps out of product crates (see AGENTS.md: `unsafe` lives only in `crates/sysmem`), so there +//! [`os_store`] answers [`CredError::Unsupported`] and callers use the file store. The platform +//! code is isolated in `platform/`, so a safe backend can drop in later. +//! - [`FileStore`]: a file encrypted with XChaCha20-Poly1305 under a key derived from the user's +//! passphrase with Argon2id. Wrong passphrases and tampering are detected, never "decrypted" +//! to garbage; writes are atomic. +//! +//! [`Secret`] holds a secret in memory: its `Debug`/`Display` never show the value and the memory +//! is zeroed on drop. Read it with [`Secret::expose`] only where it is sent. +//! +//! Native only: on wasm32 this crate is empty. + +#![cfg(not(target_arch = "wasm32"))] +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod file; +mod platform; + +pub use file::FileStore; + +use zeroize::Zeroizing; + +/// A secret value. Never printed: `{:?}` and `{}` show ``. +#[derive(Clone, PartialEq, Eq)] +pub struct Secret(Zeroizing); + +impl Secret { + pub fn new(value: impl Into) -> Secret { + Secret(Zeroizing::new(value.into())) + } + + /// The value, for the one place that sends it. + pub fn expose(&self) -> &str { + &self.0 + } +} + +impl std::fmt::Debug for Secret { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Secret()") + } +} + +impl std::fmt::Display for Secret { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("") + } +} + +/// Which secret: a service (`"immich"`) and an account within it (e.g. `server URL + user`). +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Key { + pub service: String, + pub account: String, +} + +impl Key { + pub fn new(service: &str, account: &str) -> Key { + Key { service: service.to_string(), account: account.to_string() } + } +} + +/// Why a secret could not be read or written. Messages never contain secrets. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CredError { + /// No keychain backend on this platform: use [`FileStore`]. + Unsupported(&'static str), + /// The keychain exists but can't be reached (no session bus, no Secret Service running…). + Unavailable(String), + /// The keychain is locked and unlocking needs a prompt. + Locked, + /// The passphrase does not open the file (or the file was altered). + WrongPassphrase, + /// The file is not a credentials file of a known format. + Corrupt(String), + Io(String), +} + +impl std::fmt::Display for CredError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + CredError::Unsupported(e) => write!(f, "no system keychain available: {e}"), + CredError::Unavailable(e) => write!(f, "the system keychain can't be reached: {e}"), + CredError::Locked => write!(f, "the system keychain is locked; unlock it and try again"), + CredError::WrongPassphrase => write!(f, "wrong passphrase, or the credentials file was altered"), + CredError::Corrupt(e) => write!(f, "unreadable credentials file: {e}"), + CredError::Io(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for CredError {} + +/// A place to keep secrets. +pub trait SecretStore: Send + Sync { + /// Short description for settings ("Secret Service", "encrypted file"). + fn name(&self) -> &'static str; + fn get(&self, key: &Key) -> Result, CredError>; + /// Store or replace. + fn set(&self, key: &Key, secret: &Secret) -> Result<(), CredError>; + /// `true` when something was deleted. + fn delete(&self, key: &Key) -> Result; +} + +/// The platform keychain, or why there is none. +pub fn os_store() -> Result, CredError> { + platform::open() +} + +#[cfg(test)] +mod tests; diff --git a/crates/credentials/src/platform/mod.rs b/crates/credentials/src/platform/mod.rs new file mode 100644 index 0000000000..131310915f --- /dev/null +++ b/crates/credentials/src/platform/mod.rs @@ -0,0 +1,18 @@ +//! Platform keychains, one module per backend. Everything platform-specific stays in here. + +#[cfg(all(unix, not(target_os = "macos"), not(target_os = "ios"), not(target_os = "android")))] +mod secret_service; + +use crate::{CredError, SecretStore}; + +#[cfg(all(unix, not(target_os = "macos"), not(target_os = "ios"), not(target_os = "android")))] +pub(crate) fn open() -> Result, CredError> { + Ok(Box::new(secret_service::SecretService::connect()?)) +} + +/// macOS Keychain and Windows Credential Manager are only reachable through FFI, which product +/// crates may not use (AGENTS.md). Until a safe binding exists, callers use the file store. +#[cfg(not(all(unix, not(target_os = "macos"), not(target_os = "ios"), not(target_os = "android"))))] +pub(crate) fn open() -> Result, CredError> { + Err(CredError::Unsupported("this platform's keychain needs native code; use an encrypted file")) +} diff --git a/crates/credentials/src/platform/secret_service.rs b/crates/credentials/src/platform/secret_service.rs new file mode 100644 index 0000000000..79cfaa444d --- /dev/null +++ b/crates/credentials/src/platform/secret_service.rs @@ -0,0 +1,147 @@ +//! The freedesktop Secret Service API (GNOME Keyring, KWallet, KeePassXC) over the D-Bus session +//! bus, with `zbus`'s blocking API. +//! +//! Sources: the freedesktop.org Secret Service API specification (0.2). Own implementation. +//! +//! Items carry the attributes `application` (the app id from `dac_brand`), `service` and +//! `account`; lookups match all three. The session uses the `plain` algorithm: the secret travels +//! over the local session bus to the keyring daemon, as with most clients. A locked collection +//! that needs an unlock prompt is reported as [`CredError::Locked`]. + +use std::collections::HashMap; + +use zbus::blocking::{Connection, Proxy}; +use zbus::zvariant::{ObjectPath, OwnedObjectPath, OwnedValue, Value}; + +use crate::{CredError, Key, Secret, SecretStore}; + +const DEST: &str = "org.freedesktop.secrets"; +const SERVICE_PATH: &str = "/org/freedesktop/secrets"; +const SERVICE: &str = "org.freedesktop.Secret.Service"; +const COLLECTION: &str = "org.freedesktop.Secret.Collection"; +const ITEM: &str = "org.freedesktop.Secret.Item"; +const SESSION: &str = "org.freedesktop.Secret.Session"; + +/// The Secret struct `(oayays)`: session, parameters, value, content type. +type WireSecret = (OwnedObjectPath, Vec, Vec, String); + +fn unavailable(e: zbus::Error) -> CredError { + CredError::Unavailable(e.to_string()) +} + +pub(crate) struct SecretService { + conn: Connection, +} + +/// An open `plain` session, closed on drop. +struct Session<'a> { + conn: &'a Connection, + path: OwnedObjectPath, +} + +impl Drop for Session<'_> { + fn drop(&mut self) { + if let Ok(p) = Proxy::new(self.conn, DEST, self.path.as_ref(), SESSION) { + let _ = p.call_method("Close", &()); + } + } +} + +fn is_root(p: &ObjectPath<'_>) -> bool { + p.as_str() == "/" +} + +impl SecretService { + pub(crate) fn connect() -> Result { + let conn = Connection::session().map_err(unavailable)?; + let svc = SecretService { conn }; + // fail now, not at first use, when nothing provides the service + svc.service()?.call::<_, _, OwnedObjectPath>("ReadAlias", &"default").map_err(unavailable)?; + Ok(svc) + } + + fn service(&self) -> Result, CredError> { + Proxy::new(&self.conn, DEST, SERVICE_PATH, SERVICE).map_err(unavailable) + } + + fn proxy<'a>(&'a self, path: &'a OwnedObjectPath, iface: &'static str) -> Result, CredError> { + Proxy::new(&self.conn, DEST, path.as_ref(), iface).map_err(unavailable) + } + + fn session(&self) -> Result, CredError> { + let (_, path): (OwnedValue, OwnedObjectPath) = self.service()?.call("OpenSession", &("plain", Value::from(""))).map_err(unavailable)?; + Ok(Session { conn: &self.conn, path }) + } + + fn attributes(key: &Key) -> HashMap { + HashMap::from([ + ("application".to_string(), dac_brand::APP_ID.to_string()), + ("service".to_string(), key.service.clone()), + ("account".to_string(), key.account.clone()), + ]) + } + + /// Matching items, unlocked. + fn find(&self, key: &Key) -> Result, CredError> { + let (unlocked, locked): (Vec, Vec) = + self.service()?.call("SearchItems", &Self::attributes(key)).map_err(unavailable)?; + if locked.is_empty() { + return Ok(unlocked); + } + let (mut now, prompt): (Vec, OwnedObjectPath) = self.service()?.call("Unlock", &locked).map_err(unavailable)?; + if !is_root(&prompt) && now.is_empty() { + return Err(CredError::Locked); + } + now.extend(unlocked); + Ok(now) + } +} + +impl SecretStore for SecretService { + fn name(&self) -> &'static str { + "Secret Service" + } + + fn get(&self, key: &Key) -> Result, CredError> { + let Some(item) = self.find(key)?.into_iter().next() else { return Ok(None) }; + let session = self.session()?; + let (_, _, value, _): WireSecret = self.proxy(&item, ITEM)?.call("GetSecret", &session.path).map_err(unavailable)?; + let value = zeroize::Zeroizing::new(value); + let text = std::str::from_utf8(&value).map_err(|_| CredError::Corrupt("the stored secret is not text".into()))?; + Ok(Some(Secret::new(text))) + } + + fn set(&self, key: &Key, secret: &Secret) -> Result<(), CredError> { + let collection: OwnedObjectPath = self.service()?.call("ReadAlias", &"default").map_err(unavailable)?; + if is_root(&collection) { + return Err(CredError::Unavailable("the keyring has no default collection".into())); + } + let session = self.session()?; + let label = format!("{} – {} ({})", dac_brand::DISPLAY_NAME, key.service, key.account); + let mut props: HashMap<&str, Value<'_>> = HashMap::new(); + props.insert("org.freedesktop.Secret.Item.Label", Value::from(label)); + props.insert("org.freedesktop.Secret.Item.Attributes", Value::from(Self::attributes(key))); + let wire: WireSecret = (session.path.clone(), Vec::new(), secret.expose().as_bytes().to_vec(), "text/plain; charset=utf8".into()); + let (item, prompt): (OwnedObjectPath, OwnedObjectPath) = + self.proxy(&collection, COLLECTION)?.call("CreateItem", &(props, wire, true)).map_err(unavailable)?; + if is_root(&item) && !is_root(&prompt) { + return Err(CredError::Locked); + } + log::info!("credentials: stored a secret for {} in the Secret Service", key.service); + Ok(()) + } + + fn delete(&self, key: &Key) -> Result { + let items = self.find(key)?; + for item in &items { + let prompt: OwnedObjectPath = self.proxy(item, ITEM)?.call("Delete", &()).map_err(unavailable)?; + if !is_root(&prompt) { + return Err(CredError::Locked); + } + } + if !items.is_empty() { + log::info!("credentials: removed a secret for {} from the Secret Service", key.service); + } + Ok(!items.is_empty()) + } +} diff --git a/crates/credentials/src/tests.rs b/crates/credentials/src/tests.rs new file mode 100644 index 0000000000..b5d69cb980 --- /dev/null +++ b/crates/credentials/src/tests.rs @@ -0,0 +1,135 @@ +use std::sync::Mutex; + +use super::*; + +const SECRET: &str = "immich-api-key-7f3a9c"; + +fn tempdir(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("dac-cred-test-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +struct Capture; +static LOGS: Mutex = Mutex::new(String::new()); + +impl log::Log for Capture { + fn enabled(&self, _: &log::Metadata) -> bool { + true + } + fn log(&self, r: &log::Record) { + LOGS.lock().unwrap().push_str(&format!("{} {}\n", r.target(), r.args())); + } + fn flush(&self) {} +} + +fn capture_logs() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| { + let _ = log::set_logger(&Capture); + log::set_max_level(log::LevelFilter::Trace); + }); +} + +#[test] +fn file_store_round_trip_and_wrong_passphrase() { + let dir = tempdir("rt"); + let path = dir.join("sub/credentials.json"); + let pass = Secret::new("correct horse"); + let key = Key::new("immich", "https://photos.home|me@example.org"); + { + let s = FileStore::open(&path, &pass).unwrap(); + assert_eq!(s.get(&key).unwrap(), None); + s.set(&key, &Secret::new(SECRET)).unwrap(); + s.set(&Key::new("other", "x"), &Secret::new("y")).unwrap(); + } + let s = FileStore::open(&path, &pass).unwrap(); + assert_eq!(s.get(&key).unwrap().unwrap().expose(), SECRET); + assert!(s.delete(&Key::new("other", "x")).unwrap()); + assert!(!s.delete(&Key::new("other", "x")).unwrap()); + assert_eq!(s.get(&key).unwrap().unwrap().expose(), SECRET); + + // the file never holds the plaintext + let raw = std::fs::read_to_string(&path).unwrap(); + assert!(!raw.contains(SECRET) && !raw.contains("photos.home")); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!(std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o600); + } + + assert_eq!(FileStore::open(&path, &Secret::new("wrong")).unwrap_err(), CredError::WrongPassphrase); + let _ = std::fs::remove_dir_all(dir); +} + +#[test] +fn hostile_files_are_errors() { + let dir = tempdir("hostile"); + let pass = Secret::new("p"); + let path = dir.join("c.json"); + let s = FileStore::open(&path, &pass).unwrap(); + s.set(&Key::new("a", "b"), &Secret::new("v")).unwrap(); + let good = std::fs::read_to_string(&path).unwrap(); + + // flipped ciphertext byte + let mut env: serde_json::Value = serde_json::from_str(&good).unwrap(); + let data = env["data"].as_str().unwrap().to_string(); + let flipped = format!("{}{}", if data.starts_with('0') { "1" } else { "0" }, &data[1..]); + env["data"] = flipped.into(); + std::fs::write(&path, env.to_string()).unwrap(); + assert_eq!(FileStore::open(&path, &pass).unwrap_err(), CredError::WrongPassphrase); + + // absurd KDF cost + let mut env: serde_json::Value = serde_json::from_str(&good).unwrap(); + env["m_kib"] = (u32::MAX).into(); + std::fs::write(&path, env.to_string()).unwrap(); + assert!(matches!(FileStore::open(&path, &pass), Err(CredError::Corrupt(_)))); + + for junk in ["", "{}", "not json", r#"{"format":1,"kdf":"argon2id","m_kib":64,"t":1,"p":1,"salt":"zz","nonce":"","data":""}"#] { + std::fs::write(&path, junk).unwrap(); + assert!(FileStore::open(&path, &pass).is_err(), "{junk}"); + } + let _ = std::fs::remove_dir_all(dir); +} + +#[test] +fn secrets_never_reach_debug_output_or_logs() { + capture_logs(); + let secret = Secret::new(SECRET); + assert_eq!(format!("{secret:?}"), "Secret()"); + assert_eq!(format!("{secret}"), ""); + + let dir = tempdir("logs"); + let path = dir.join("c.json"); + let s = FileStore::open(&path, &Secret::new("passphrase-xyz")).unwrap(); + let key = Key::new("immich", "acct"); + s.set(&key, &secret).unwrap(); + let got = s.get(&key).unwrap(); + s.delete(&key).unwrap(); + let wrong = FileStore::open(&path, &Secret::new("nope")).err(); + let debug = format!("{s:?} {got:?} {wrong:?}"); + // the OS store may or may not exist here; either way its output must not leak + let os = os_store().map(|st| st.name()).map_err(|e| e.to_string()); + let logs = LOGS.lock().unwrap().clone(); + assert!(logs.contains("credentials: stored a secret"), "logging works: {logs}"); + for text in [&logs, &debug, &format!("{os:?}")] { + assert!(!text.contains(SECRET), "secret leaked into {text}"); + assert!(!text.contains("passphrase-xyz"), "passphrase leaked into {text}"); + } + let _ = std::fs::remove_dir_all(dir); +} + +/// Exercises the real keychain only when asked (it may prompt or touch the user's keyring). +#[test] +fn os_store_round_trip_when_enabled() { + if std::env::var_os("DAC_TEST_OS_KEYCHAIN").is_none() { + return; + } + let store = os_store().unwrap(); + let key = Key::new("test-service", &format!("test-{}", std::process::id())); + store.set(&key, &Secret::new(SECRET)).unwrap(); + assert_eq!(store.get(&key).unwrap().unwrap().expose(), SECRET); + assert!(store.delete(&key).unwrap()); + assert_eq!(store.get(&key).unwrap(), None); +} diff --git a/crates/hash/Cargo.toml b/crates/hash/Cargo.toml new file mode 100644 index 0000000000..fffc4ec9d4 --- /dev/null +++ b/crates/hash/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "dac-hash" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Streaming content hashes (L0): SHA-1 of originals, as remote services such as Immich store them" + +[dependencies] +sha1 = { version = "0.10", default-features = false } + +[lints] +workspace = true diff --git a/crates/hash/src/lib.rs b/crates/hash/src/lib.rs new file mode 100644 index 0000000000..e57856d377 --- /dev/null +++ b/crates/hash/src/lib.rs @@ -0,0 +1,167 @@ +//! Streaming SHA-1 of a file's bytes, for matching originals with remote services that identify +//! assets by it (Immich stores the SHA-1 of every original). SHA-1 is used here as a content +//! identifier agreed with those services, not for security. +//! +//! Made to ride along an existing read pass: wrap the reader the import already uses in a +//! [`HashingReader`] (or feed the chunks it reads to a [`Sha1`]) and take the digest at the end, +//! so a file is read from disk once. [`sha1_reader`] and [`sha1_file`] cover a standalone pass +//! (back-filling old catalogs). +//! +//! Pure computation; builds everywhere, wasm32 included. + +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +use std::io::Read; +use std::path::Path; + +use sha1::Digest; + +/// A SHA-1 digest. +#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct Sha1Digest(pub [u8; 20]); + +impl Sha1Digest { + /// Lower-case hex (40 characters). + pub fn to_hex(&self) -> String { + self.0.iter().map(|b| format!("{b:02x}")).collect() + } + + /// Parse 40 hex characters (any case). + pub fn from_hex(s: &str) -> Option { + let s = s.trim(); + if s.len() != 40 { + return None; + } + let mut out = [0u8; 20]; + for (i, b) in out.iter_mut().enumerate() { + *b = u8::from_str_radix(s.get(i * 2..i * 2 + 2)?, 16).ok()?; + } + Some(Sha1Digest(out)) + } + + /// Standard base64, the form Immich's API uses for `checksum`. + pub fn to_base64(&self) -> String { + const A: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + let mut out = String::with_capacity(28); + for chunk in self.0.chunks(3) { + let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)]; + let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]); + for i in 0..4 { + if i <= chunk.len() { + out.push(char::from(A.get(((n >> (18 - 6 * i)) & 63) as usize).copied().unwrap_or(b'='))); + } else { + out.push('='); + } + } + } + out + } +} + +impl std::fmt::Display for Sha1Digest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.to_hex()) + } +} + +impl std::fmt::Debug for Sha1Digest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Sha1Digest({})", self.to_hex()) + } +} + +/// An incremental SHA-1. +#[derive(Clone, Default)] +pub struct Sha1(sha1::Sha1); + +impl Sha1 { + pub fn new() -> Sha1 { + Sha1::default() + } + + pub fn update(&mut self, bytes: &[u8]) { + self.0.update(bytes); + } + + pub fn finish(self) -> Sha1Digest { + Sha1Digest(self.0.finalize().into()) + } +} + +/// Hashes everything read through it. +pub struct HashingReader { + inner: R, + hasher: Sha1, + bytes: u64, +} + +impl HashingReader { + pub fn new(inner: R) -> Self { + HashingReader { inner, hasher: Sha1::new(), bytes: 0 } + } + + /// Bytes read so far. + pub fn bytes_read(&self) -> u64 { + self.bytes + } + + /// The digest of everything read so far, and the reader back. Read to the end first for the + /// whole file's SHA-1. + pub fn finish(self) -> (R, Sha1Digest) { + (self.inner, self.hasher.finish()) + } +} + +impl Read for HashingReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let n = self.inner.read(buf)?; + self.hasher.update(buf.get(..n).unwrap_or_default()); + self.bytes = self.bytes.saturating_add(n as u64); + Ok(n) + } +} + +/// SHA-1 of everything `r` yields. +pub fn sha1_reader(r: impl Read) -> std::io::Result { + let mut h = HashingReader::new(r); + std::io::copy(&mut h, &mut std::io::sink())?; + Ok(h.finish().1) +} + +/// SHA-1 of a file's bytes. +pub fn sha1_file(path: &Path) -> std::io::Result { + sha1_reader(std::io::BufReader::with_capacity(1 << 20, std::fs::File::open(path)?)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn known_vectors() { + assert_eq!(sha1_reader(&b""[..]).unwrap().to_hex(), "da39a3ee5e6b4b0d3255bfef95601890afd80709"); + let abc = sha1_reader(&b"abc"[..]).unwrap(); + assert_eq!(abc.to_hex(), "a9993e364706816aba3e25717850c26c9cd0d89d"); + assert_eq!(abc.to_base64(), "qZk+NkcGgWq6PiVxeFDCbJzQ2J0="); + assert_eq!(Sha1Digest::from_hex(&abc.to_hex().to_uppercase()), Some(abc)); + assert_eq!(Sha1Digest::from_hex("xyz"), None); + assert_eq!(Sha1Digest::from_hex(&"é".repeat(20)), None); + } + + #[test] + fn hashing_reader_rides_along() { + let data: Vec = (0..100_000u32).map(|i| i as u8).collect(); + let mut r = HashingReader::new(&data[..]); + let mut copy = Vec::new(); + r.read_to_end(&mut copy).unwrap(); + assert_eq!(r.bytes_read(), 100_000); + let (_, d) = r.finish(); + let mut h = Sha1::new(); + for c in data.chunks(777) { + h.update(c); + } + assert_eq!(h.finish(), d); + assert_eq!(copy, data); + } +} diff --git a/crates/net/Cargo.toml b/crates/net/Cargo.toml new file mode 100644 index 0000000000..ec38da4d99 --- /dev/null +++ b/crates/net/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "dac-net" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "HTTP(S) client in pure Rust (L0): JSON, multipart upload, resumable verified downloads, proxy, per-connection trust" + +[dependencies] +dac-brand = { workspace = true } + +[lints] +workspace = true + +# Native only: on wasm32 the crate is empty (the web build makes no requests through it). +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +log = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +rustls = { version = "0.23", default-features = false, features = ["std", "tls12"] } +rustls-rustcrypto = "0.0.2-alpha" +webpki-roots = "1" +sha2 = { version = "0.10", default-features = false, features = ["std"] } +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } diff --git a/crates/net/src/client.rs b/crates/net/src/client.rs new file mode 100644 index 0000000000..299f460e3a --- /dev/null +++ b/crates/net/src/client.rs @@ -0,0 +1,414 @@ +//! The client: configuration, the request builder and the response. + +use std::io::{BufReader, BufWriter, Read, Write}; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; +use std::time::Duration; + +use serde::Serialize; +use serde::de::DeserializeOwned; + +use crate::conn::{self, ConnectOptions}; +use crate::http::{self, BodyReader, Framing}; +use crate::{Multipart, NetError, Trust, Url}; + +/// HTTP method. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Method { + Get, + Head, + Post, + Put, + Patch, + Delete, +} + +impl Method { + pub fn as_str(self) -> &'static str { + match self { + Method::Get => "GET", + Method::Head => "HEAD", + Method::Post => "POST", + Method::Put => "PUT", + Method::Patch => "PATCH", + Method::Delete => "DELETE", + } + } +} + +/// Header names that carry credentials: never sent to another origin after a redirect. +const SENSITIVE: &[&str] = &["authorization", "proxy-authorization", "cookie", "x-api-key"]; + +/// Settings shared by every request of a [`Client`]. +#[derive(Debug, Clone)] +pub struct ClientConfig { + /// Per address tried. + pub connect_timeout: Duration, + /// Longest wait for any progress while sending or receiving. + pub stall_timeout: Duration, + /// Redirects followed per request (0: none, the 3xx response is returned). + pub max_redirects: u8, + /// `http://host:port` of an HTTP proxy (HTTPS goes through it with `CONNECT`). + pub proxy: Option, + /// Which server certificates are trusted. + pub trust: Trust, + /// Cap for [`Response::bytes`], [`Response::text`] and [`Response::json`]. + pub max_body: u64, +} + +impl Default for ClientConfig { + fn default() -> Self { + ClientConfig { + connect_timeout: Duration::from_secs(10), + stall_timeout: Duration::from_secs(30), + max_redirects: 5, + proxy: None, + trust: Trust::new(), + max_body: 64 * 1024 * 1024, + } + } +} + +/// An HTTP(S) client. Blocking: run it off the UI thread. Cheap to clone. +#[derive(Clone)] +pub struct Client { + config: ClientConfig, + proxy: Option, + tls: Arc, +} + +impl std::fmt::Debug for Client { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Client").field("config", &self.config).finish() + } +} + +impl Client { + pub fn new(config: ClientConfig) -> Result { + let proxy = match &config.proxy { + Some(p) => { + let u = Url::parse(p)?; + if u.tls { + return Err(NetError::BadUrl("only http:// proxies are supported".into())); + } + Some(u) + } + None => None, + }; + let tls = config.trust.client_config()?; + Ok(Client { config, proxy, tls }) + } + + pub fn config(&self) -> &ClientConfig { + &self.config + } + + pub fn request(&self, method: Method, url: &str) -> Request<'_> { + Request { + client: self, + method, + url: Url::parse(url), + headers: Vec::new(), + secret_headers: Vec::new(), + body: Body::Empty, + cancel: None, + progress: None, + error: None, + accept_gzip: true, + } + } + + pub fn get(&self, url: &str) -> Request<'_> { + self.request(Method::Get, url) + } + pub fn head(&self, url: &str) -> Request<'_> { + self.request(Method::Head, url) + } + pub fn post(&self, url: &str) -> Request<'_> { + self.request(Method::Post, url) + } + pub fn put(&self, url: &str) -> Request<'_> { + self.request(Method::Put, url) + } + pub fn patch(&self, url: &str) -> Request<'_> { + self.request(Method::Patch, url) + } + pub fn delete(&self, url: &str) -> Request<'_> { + self.request(Method::Delete, url) + } +} + +enum Body { + Empty, + Bytes { data: Vec, content_type: String }, + Multipart(Multipart), +} + +/// Progress callback: `(bytes sent, total)` while uploading. +pub type Progress<'a> = &'a mut dyn FnMut(u64, Option); + +/// A request under construction; [`Request::send`] performs it. +pub struct Request<'a> { + client: &'a Client, + method: Method, + url: Result, + headers: Vec<(String, String)>, + secret_headers: Vec, + body: Body, + cancel: Option>, + progress: Option>, + error: Option, + accept_gzip: bool, +} + +fn check_header(name: &str, value: &str) -> Result<(), NetError> { + let bad_name = name.is_empty() || name.chars().any(|c| !c.is_ascii_graphic() || c == ':'); + if bad_name || value.chars().any(|c| c == '\r' || c == '\n' || c == '\0') { + // the value is never put in the message: it may be a secret + return Err(NetError::BadHeader(format!("{:?}", name.chars().take(64).collect::()))); + } + Ok(()) +} + +impl<'a> Request<'a> { + /// Add a header. + pub fn header(mut self, name: &str, value: &str) -> Self { + match check_header(name, value) { + Ok(()) => self.headers.push((name.to_string(), value.to_string())), + Err(e) => self.error = self.error.or(Some(e)), + } + self + } + + /// Add a header holding a credential: it is dropped when a redirect leaves the origin. + pub fn secret_header(mut self, name: &str, value: &str) -> Self { + self.secret_headers.push(name.to_ascii_lowercase()); + self.header(name, value) + } + + /// `Authorization: Bearer …`. + pub fn bearer(self, token: &str) -> Self { + self.secret_header("Authorization", &format!("Bearer {token}")) + } + + /// A raw body. + pub fn body(mut self, content_type: &str, data: Vec) -> Self { + self.body = Body::Bytes { data, content_type: content_type.to_string() }; + self + } + + /// A JSON body. + pub fn json(mut self, value: &T) -> Self { + match serde_json::to_vec(value) { + Ok(data) => self.body = Body::Bytes { data, content_type: "application/json".into() }, + Err(e) => self.error = self.error.or(Some(NetError::Json(e.to_string()))), + } + self + } + + /// A streaming `multipart/form-data` body. + pub fn multipart(mut self, form: Multipart) -> Self { + self.body = Body::Multipart(form); + self + } + + /// Abort (with [`NetError::Cancelled`]) once this flag is raised, also while reading the + /// response body. + pub fn cancel(mut self, flag: Arc) -> Self { + self.cancel = Some(flag); + self + } + + /// Report upload progress. + pub fn progress(mut self, f: Progress<'a>) -> Self { + self.progress = Some(f); + self + } + + /// Don't ask for a compressed response (downloads with `Range`). + pub fn identity_encoding(mut self) -> Self { + self.accept_gzip = false; + self + } + + /// Perform the request, following redirects. Any status is a successful `send`; see + /// [`Response::error_for_status`]. + pub fn send(self) -> Result { + let Request { client, mut method, url, mut headers, secret_headers, mut body, cancel, mut progress, error, accept_gzip } = self; + if let Some(e) = error { + return Err(e); + } + let mut url = url?; + let origin = url.origin(); + let mut redirects = 0u8; + loop { + let path_only = url.path.split('?').next().unwrap_or("/"); + log::debug!("net: {} {}{}", method.as_str(), url.origin(), path_only); + let opts = ConnectOptions { + connect_timeout: client.config.connect_timeout, + stall: client.config.stall_timeout, + cancel: cancel.clone(), + proxy: client.proxy.as_ref(), + tls: Some(client.tls.clone()), + }; + let conn = conn::connect(&url, opts)?; + let mut w = BufWriter::with_capacity(64 * 1024, conn); + let target = if client.proxy.is_some() && !url.tls { url.to_string() } else { url.path.clone() }; + let mut head = format!( + "{} {target} HTTP/1.1\r\nHost: {}\r\nUser-Agent: {}\r\nConnection: close\r\n", + method.as_str(), + url.host_header(), + crate::user_agent() + ); + let has = |n: &str| headers.iter().any(|(k, _)| k.eq_ignore_ascii_case(n)); + if !has("accept") { + head.push_str("Accept: */*\r\n"); + } + if !has("accept-encoding") { + head.push_str(if accept_gzip { "Accept-Encoding: gzip\r\n" } else { "Accept-Encoding: identity\r\n" }); + } + let total = match &body { + Body::Empty => (method != Method::Get && method != Method::Head && method != Method::Delete).then_some(0), + Body::Bytes { data, .. } => Some(data.len() as u64), + Body::Multipart(m) => Some(m.len()), + }; + if let Some(n) = total { + head.push_str(&format!("Content-Length: {n}\r\n")); + } + match &body { + Body::Bytes { content_type, .. } if !has("content-type") => head.push_str(&format!("Content-Type: {content_type}\r\n")), + Body::Multipart(m) => head.push_str(&format!("Content-Type: {}\r\n", m.content_type())), + _ => {} + } + for (k, v) in &headers { + head.push_str(&format!("{k}: {v}\r\n")); + } + head.push_str("\r\n"); + w.write_all(head.as_bytes())?; + let replay = match std::mem::replace(&mut body, Body::Empty) { + Body::Empty => Some(Body::Empty), + Body::Bytes { data, content_type } => { + for chunk in data.chunks(64 * 1024) { + w.write_all(chunk)?; + } + if let Some(p) = progress.as_mut() { + p(data.len() as u64, total); + } + Some(Body::Bytes { data, content_type }) + } + Body::Multipart(m) => { + let mut report = |sent: u64| { + if let Some(p) = progress.as_mut() { + p(sent, total); + } + }; + m.write_to(&mut w, &mut report)?; + None + } + }; + w.flush()?; + let conn = w.into_inner().map_err(|e| NetError::Io(e.to_string()))?; + let mut r = BufReader::with_capacity(64 * 1024, conn); + let head = http::read_head(&mut r)?; + + let location = head.header("location").filter(|_| matches!(head.status, 301 | 302 | 303 | 307 | 308)); + if let Some(location) = location { + if redirects >= client.config.max_redirects { + if client.config.max_redirects == 0 { + return Response::new(head, r, url, method, client.config.max_body); + } + return Err(NetError::Redirect(format!("more than {} redirects", client.config.max_redirects))); + } + redirects += 1; + let next = url.join(location)?; + if url.tls && !next.tls { + return Err(NetError::Redirect("refusing to follow a redirect from https to http".into())); + } + if head.status == 303 || ((head.status == 301 || head.status == 302) && method == Method::Post) { + method = if method == Method::Head { Method::Head } else { Method::Get }; + headers.retain(|(k, _)| !k.eq_ignore_ascii_case("content-type")); + body = Body::Empty; + } else { + body = replay.ok_or_else(|| NetError::Redirect("a streamed upload can't be resent to the redirect target".into()))?; + } + if next.origin() != origin { + headers.retain(|(k, _)| { + let k = k.to_ascii_lowercase(); + !SENSITIVE.contains(&k.as_str()) && !secret_headers.contains(&k) + }); + } + url = next; + continue; + } + return Response::new(head, r, url, method, client.config.max_body); + } + } +} + +/// A response: status and headers, and the body as a stream ([`Read`]). +pub struct Response { + pub status: u16, + pub reason: String, + pub headers: Vec<(String, String)>, + /// Where the response came from (after redirects). + pub url: Url, + body: Box, + max_body: u64, +} + +impl std::fmt::Debug for Response { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // headers can carry cookies: only their names + let names: Vec<&str> = self.headers.iter().map(|(k, _)| k.as_str()).collect(); + f.debug_struct("Response").field("status", &self.status).field("url", &self.url.origin()).field("headers", &names).finish() + } +} + +impl Response { + fn new(head: http::Head, r: BufReader, url: Url, method: Method, max_body: u64) -> Result { + let framing = Framing::of(&head, method == Method::Head)?; + let raw = BodyReader::new(r, framing); + let gzip = framing != Framing::Empty && head.header("content-encoding").is_some_and(|e| e.trim().eq_ignore_ascii_case("gzip")); + let body: Box = if gzip { Box::new(flate2::read::GzDecoder::new(raw)) } else { Box::new(raw) }; + Ok(Response { status: head.status, reason: head.reason, headers: head.headers, url, body, max_body }) + } + + /// First value of a header (case-insensitive). + pub fn header(&self, name: &str) -> Option<&str> { + self.headers.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str()) + } + + pub fn is_success(&self) -> bool { + (200..300).contains(&self.status) + } + + /// `Err(Status)` for a non-2xx response. + pub fn error_for_status(self) -> Result { + if self.is_success() { Ok(self) } else { Err(NetError::Status { code: self.status, reason: self.reason.clone() }) } + } + + /// The whole body (at most the client's `max_body`). + pub fn bytes(mut self) -> Result, NetError> { + let cap = self.max_body; + let mut out = Vec::new(); + (&mut self.body).take(cap.saturating_add(1)).read_to_end(&mut out)?; + if out.len() as u64 > cap { + return Err(NetError::TooLarge(cap)); + } + Ok(out) + } + + pub fn text(self) -> Result { + Ok(String::from_utf8_lossy(&self.bytes()?).into_owned()) + } + + pub fn json(self) -> Result { + serde_json::from_slice(&self.bytes()?).map_err(|e| NetError::Json(e.to_string())) + } +} + +impl Read for Response { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.body.read(buf) + } +} diff --git a/crates/net/src/conn.rs b/crates/net/src/conn.rs new file mode 100644 index 0000000000..aa7e238a22 --- /dev/null +++ b/crates/net/src/conn.rs @@ -0,0 +1,192 @@ +//! One connection: TCP (with connect and stall timeouts and a cancel flag), optionally through an +//! HTTP proxy, optionally wrapped in TLS. + +use std::io::{ErrorKind, Read, Write}; +use std::net::{TcpStream, ToSocketAddrs}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::{Duration, Instant}; + +use crate::{NetError, Url}; + +/// How often a blocked read or write wakes up to look at the cancel flag. +const POLL: Duration = Duration::from_millis(250); + +/// A TCP stream whose reads and writes only return data or a real error: they wait through +/// socket timeouts, fail with [`NetError::Stalled`] after `stall` without progress and with +/// [`NetError::Cancelled`] once the flag is raised. +pub(crate) struct TimedTcp { + tcp: TcpStream, + stall: Duration, + cancel: Option>, +} + +impl TimedTcp { + fn check_cancel(&self) -> std::io::Result<()> { + match &self.cancel { + Some(c) if c.load(Ordering::Relaxed) => Err(NetError::Cancelled.into()), + _ => Ok(()), + } + } +} + +fn is_timeout(e: &std::io::Error) -> bool { + matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut | ErrorKind::Interrupted) +} + +impl Read for TimedTcp { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let start = Instant::now(); + loop { + self.check_cancel()?; + match self.tcp.read(buf) { + Err(e) if is_timeout(&e) => { + if start.elapsed() >= self.stall { + return Err(NetError::Stalled.into()); + } + } + other => return other, + } + } + } +} + +impl Write for TimedTcp { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let start = Instant::now(); + loop { + self.check_cancel()?; + match self.tcp.write(buf) { + Err(e) if is_timeout(&e) => { + if start.elapsed() >= self.stall { + return Err(NetError::Stalled.into()); + } + } + other => return other, + } + } + } + + fn flush(&mut self) -> std::io::Result<()> { + self.tcp.flush() + } +} + +/// A plain or TLS connection. +pub(crate) enum Conn { + Plain(TimedTcp), + Tls(Box>), +} + +impl Read for Conn { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + Conn::Plain(s) => s.read(buf), + Conn::Tls(s) => match s.read(buf) { + // servers that close without close_notify: treat as end of stream (the body + // framing, Content-Length or chunked, still catches truncation) + Err(e) if e.kind() == ErrorKind::UnexpectedEof => Ok(0), + other => other, + }, + } + } +} + +impl Write for Conn { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + match self { + Conn::Plain(s) => s.write(buf), + Conn::Tls(s) => s.write(buf), + } + } + + fn flush(&mut self) -> std::io::Result<()> { + match self { + Conn::Plain(s) => s.flush(), + Conn::Tls(s) => s.flush(), + } + } +} + +pub(crate) struct ConnectOptions<'a> { + pub connect_timeout: Duration, + pub stall: Duration, + pub cancel: Option>, + pub proxy: Option<&'a Url>, + pub tls: Option>, +} + +fn tcp_connect(host: &str, port: u16, timeout: Duration, cancel: &Option>) -> Result { + let addrs: Vec<_> = (host, port).to_socket_addrs().map_err(|e| NetError::Connect(format!("{host}: {e}")))?.collect(); + let mut last = format!("{host}: no address"); + for addr in addrs { + if cancel.as_ref().is_some_and(|c| c.load(Ordering::Relaxed)) { + return Err(NetError::Cancelled); + } + match TcpStream::connect_timeout(&addr, timeout) { + Ok(s) => return Ok(s), + Err(e) => last = format!("{host} ({addr}): {e}"), + } + } + Err(NetError::Connect(last)) +} + +/// Open a connection to `url`'s origin. Through a proxy, `http://` targets talk to the proxy +/// directly (the request line then carries the absolute URL) and `https://` targets are tunnelled +/// with `CONNECT`. +pub(crate) fn connect(url: &Url, opts: ConnectOptions<'_>) -> Result { + let (host, port) = match opts.proxy { + Some(p) => (p.host.as_str(), p.port), + None => (url.host.as_str(), url.port), + }; + let tcp = tcp_connect(host, port, opts.connect_timeout, &opts.cancel)?; + let _ = tcp.set_nodelay(true); + tcp.set_read_timeout(Some(POLL)).map_err(|e| NetError::Io(e.to_string()))?; + tcp.set_write_timeout(Some(POLL)).map_err(|e| NetError::Io(e.to_string()))?; + let mut timed = TimedTcp { tcp, stall: opts.stall, cancel: opts.cancel }; + if opts.proxy.is_some() && url.tls { + tunnel(&mut timed, url)?; + } + let Some(config) = opts.tls.filter(|_| url.tls) else { + return Ok(Conn::Plain(timed)); + }; + let name = rustls::pki_types::ServerName::try_from(url.host.clone()).map_err(|e| NetError::Tls(format!("{}: {e}", url.host)))?; + let mut conn = rustls::ClientConnection::new(config, name).map_err(|e| NetError::Tls(e.to_string()))?; + // finish the handshake now so certificate errors surface as such + while conn.is_handshaking() { + conn.complete_io(&mut timed).map_err(tls_error)?; + } + Ok(Conn::Tls(Box::new(rustls::StreamOwned::new(conn, timed)))) +} + +/// Map a handshake I/O error, unwrapping our own errors (untrusted certificate, stall, cancel). +fn tls_error(e: std::io::Error) -> NetError { + if let Some(inner) = e.get_ref() { + if let Some(n) = inner.downcast_ref::() { + return n.clone(); + } + if let Some(rustls::Error::InvalidCertificate(rustls::CertificateError::Other(o))) = inner.downcast_ref::() + && let Some(n) = o.0.downcast_ref::() + { + return n.clone(); + } + if let Some(rustls::Error::Other(o)) = inner.downcast_ref::() + && let Some(n) = o.0.downcast_ref::() + { + return n.clone(); + } + } + NetError::Tls(e.to_string()) +} + +/// `CONNECT host:port` through the proxy; any 2xx opens the tunnel. +fn tunnel(s: &mut TimedTcp, url: &Url) -> Result<(), NetError> { + let target = if url.host.contains(':') { format!("[{}]:{}", url.host, url.port) } else { format!("{}:{}", url.host, url.port) }; + let req = format!("CONNECT {target} HTTP/1.1\r\nHost: {target}\r\nUser-Agent: {}\r\n\r\n", crate::user_agent()); + s.write_all(req.as_bytes())?; + let head = crate::http::read_head(s)?; + if !(200..300).contains(&head.status) { + return Err(NetError::Connect(format!("proxy refused the tunnel: {} {}", head.status, head.reason))); + } + Ok(()) +} diff --git a/crates/net/src/download.rs b/crates/net/src/download.rs new file mode 100644 index 0000000000..c7d3a3f8c1 --- /dev/null +++ b/crates/net/src/download.rs @@ -0,0 +1,195 @@ +//! Streaming a download to a file: resumable (`.part` plus `Range`), size-capped, and +//! checked against an expected size and SHA-256 before it is renamed into place. A file that +//! fails its check is deleted, never kept under its real name. + +use std::fs::{File, OpenOptions}; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use sha2::{Digest, Sha256}; + +use crate::{Client, NetError}; + +/// What [`Client::download`] checks and reports. +#[derive(Default)] +pub struct DownloadOptions<'a> { + /// Exact size in bytes, when known. + pub size: Option, + /// SHA-256 (hex, any case), when known. + pub sha256: Option, + /// Refuse anything larger (0: no cap besides `size`). + pub max_size: u64, + /// Extra headers (`secret`: dropped on cross-origin redirects). + pub headers: Vec<(String, String, bool)>, + pub cancel: Option>, + /// `(bytes so far, total when known)`. + pub progress: Option<&'a mut dyn FnMut(u64, Option)>, +} + +/// A finished download. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Downloaded { + pub bytes: u64, + /// Lower-case hex SHA-256 of the file. + pub sha256: String, +} + +fn part_path(dest: &Path) -> PathBuf { + let mut name = dest.file_name().map(|n| n.to_os_string()).unwrap_or_default(); + name.push(".part"); + dest.with_file_name(name) +} + +fn io(path: &Path, e: std::io::Error) -> NetError { + NetError::Io(format!("{}: {e}", path.display())) +} + +/// Parse `Content-Range: bytes START-END/TOTAL` → (start, total). +fn content_range(v: &str) -> Option<(u64, Option)> { + let rest = v.trim().strip_prefix("bytes")?.trim(); + let (range, total) = rest.split_once('/')?; + let start = range.split_once('-')?.0.trim().parse().ok()?; + Some((start, total.trim().parse().ok())) +} + +impl Client { + /// Download `url` to `dest`, resuming a previous `.part`. + pub fn download(&self, url: &str, dest: &Path, mut opts: DownloadOptions<'_>) -> Result { + let part = part_path(dest); + let cap = match (opts.size, opts.max_size) { + (Some(s), _) => s, + (None, 0) => u64::MAX, + (None, m) => m, + }; + // hash what is already there + let mut hasher = Sha256::new(); + let mut have = 0u64; + if let Ok(mut f) = File::open(&part) { + let mut buf = vec![0u8; 256 * 1024]; + loop { + let n = f.read(&mut buf).map_err(|e| io(&part, e))?; + if n == 0 { + break; + } + hasher.update(buf.get(..n).unwrap_or_default()); + have = have.saturating_add(n as u64); + } + } + if have > cap { + // left over from something else: start again + have = 0; + hasher = Sha256::new(); + } + + let mut req = self.get(url).identity_encoding(); + for (k, v, secret) in &opts.headers { + req = if *secret { req.secret_header(k, v) } else { req.header(k, v) }; + } + if let Some(c) = &opts.cancel { + req = req.cancel(c.clone()); + } + let done_already = opts.size.is_some_and(|s| s == have && s > 0); + if have > 0 && !done_already { + req = req.header("Range", &format!("bytes={have}-")); + } + let (mut resp, mut file) = if done_already { + (None, None) + } else { + let resp = req.send()?; + let append = match resp.status { + 206 if have > 0 => match resp.header("content-range").and_then(content_range) { + Some((start, _)) if start == have => true, + _ => return Err(NetError::Protocol("server resumed at the wrong offset".into())), + }, + 200 => false, + 416 if have > 0 => { + let _ = std::fs::remove_file(&part); + return Err(NetError::Protocol("server refused to resume; the partial file was removed, try again".into())); + } + code => return Err(NetError::Status { code, reason: resp.reason.clone() }), + }; + if !append { + have = 0; + hasher = Sha256::new(); + } + if let Some(parent) = part.parent().filter(|p| !p.as_os_str().is_empty()) { + std::fs::create_dir_all(parent).map_err(|e| io(parent, e))?; + } + let file = OpenOptions::new().create(true).write(true).append(append).truncate(!append).open(&part).map_err(|e| io(&part, e))?; + (Some(resp), Some(file)) + }; + + let total = opts.size.or_else(|| { + let r = resp.as_ref()?; + let len: u64 = r.header("content-length")?.trim().parse().ok()?; + Some(len.saturating_add(have)) + }); + if let (Some(t), true) = (total, cap != u64::MAX) + && t > cap + { + return Err(NetError::TooLarge(cap)); + } + let mut too_large = false; + if let (Some(resp), Some(file)) = (resp.as_mut(), file.as_mut()) { + let mut buf = vec![0u8; 256 * 1024]; + loop { + let n = resp.read(&mut buf)?; + if n == 0 { + break; + } + let chunk = buf.get(..n).unwrap_or_default(); + have = have.saturating_add(n as u64); + if have > cap { + too_large = true; + break; + } + file.write_all(chunk).map_err(|e| io(&part, e))?; + hasher.update(chunk); + if let Some(p) = opts.progress.as_mut() { + p(have, total); + } + } + if !too_large { + file.sync_all().map_err(|e| io(&part, e))?; + } + } + drop(file); + if too_large { + let _ = std::fs::remove_file(&part); + return Err(NetError::TooLarge(cap)); + } + + let sha = hasher.finalize().iter().map(|b| format!("{b:02x}")).collect::(); + let fail = |why: String| { + let _ = std::fs::remove_file(&part); + Err(NetError::Verify(why)) + }; + if let Some(size) = opts.size + && size != have + { + return fail(format!("expected {size} bytes, got {have}")); + } + if let Some(want) = &opts.sha256 + && !want.trim().eq_ignore_ascii_case(&sha) + { + return fail("SHA-256 does not match".into()); + } + std::fs::rename(&part, dest).map_err(|e| io(dest, e))?; + Ok(Downloaded { bytes: have, sha256: sha }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_content_range() { + assert_eq!(content_range("bytes 10-99/100"), Some((10, Some(100)))); + assert_eq!(content_range("bytes 10-99/*"), Some((10, None))); + assert_eq!(content_range("items 1-2/3"), None); + assert_eq!(part_path(Path::new("/a/b.bin")), PathBuf::from("/a/b.bin.part")); + } +} diff --git a/crates/net/src/error.rs b/crates/net/src/error.rs new file mode 100644 index 0000000000..5774bbf1cf --- /dev/null +++ b/crates/net/src/error.rs @@ -0,0 +1,78 @@ +//! The one error type of the crate. + +/// Why a request failed. Messages never contain header values, bodies or credentials. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NetError { + /// The URL can't be used (scheme, host, characters). + BadUrl(String), + /// A header name or value can't be sent (line breaks, colons in names). + BadHeader(String), + /// Name resolution or connecting failed, or took longer than the connect timeout. + Connect(String), + /// No data moved for the stall timeout. + Stalled, + /// The cancel flag was raised. + Cancelled, + /// TLS failed (handshake, protocol). + Tls(String), + /// The server's certificate is not trusted and nobody accepted it (no trust-on-first-use hook, + /// or the hook declined). Carries the SHA-256 fingerprint so the UI can show it. + UntrustedCertificate { host: String, fingerprint: String }, + /// The server's response could not be understood. + Protocol(String), + /// More redirects than allowed, or a redirect that a non-replayable body can't follow. + Redirect(String), + /// The response (or a file) is larger than the allowed size. + TooLarge(u64), + /// The server answered with an error status ([`crate::Response::error_for_status`]). + Status { code: u16, reason: String }, + /// A JSON body could not be written or read. + Json(String), + /// A downloaded file does not match its expected size or SHA-256. + Verify(String), + /// Any other I/O error (local files included). + Io(String), +} + +impl std::fmt::Display for NetError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + NetError::BadUrl(e) => write!(f, "unusable URL: {e}"), + NetError::BadHeader(e) => write!(f, "unusable header: {e}"), + NetError::Connect(e) => write!(f, "could not connect: {e}"), + NetError::Stalled => write!(f, "the server stopped responding"), + NetError::Cancelled => write!(f, "cancelled"), + NetError::Tls(e) => write!(f, "secure connection failed: {e}"), + NetError::UntrustedCertificate { host, fingerprint } => { + write!(f, "the certificate of {host} is not trusted (SHA-256 fingerprint {fingerprint})") + } + NetError::Protocol(e) => write!(f, "unexpected response: {e}"), + NetError::Redirect(e) => write!(f, "redirect not followed: {e}"), + NetError::TooLarge(n) => write!(f, "larger than the allowed {n} bytes"), + NetError::Status { code, reason } => write!(f, "server answered {code} {reason}"), + NetError::Json(e) => write!(f, "JSON: {e}"), + NetError::Verify(e) => write!(f, "verification failed: {e}"), + NetError::Io(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for NetError {} + +impl From for NetError { + fn from(e: std::io::Error) -> Self { + // the stream wrappers carry our own errors through io::Error (rustls, flate2) + if let Some(inner) = e.get_ref().and_then(|i| i.downcast_ref::()) { + return inner.clone(); + } + NetError::Io(e.to_string()) + } +} + +impl From for std::io::Error { + fn from(e: NetError) -> Self { + // never `Interrupted`: std retries those, which would swallow a cancel + let kind = if e == NetError::Stalled { std::io::ErrorKind::TimedOut } else { std::io::ErrorKind::Other }; + std::io::Error::new(kind, e) + } +} diff --git a/crates/net/src/http.rs b/crates/net/src/http.rs new file mode 100644 index 0000000000..b23e62aeae --- /dev/null +++ b/crates/net/src/http.rs @@ -0,0 +1,246 @@ +//! HTTP/1.1 wire format: the response head and body framing (Content-Length, chunked, or read +//! until close). Everything read from the server is bounded. + +use std::io::{BufRead, Read}; + +use crate::NetError; + +/// Largest response head (status line plus headers). +const MAX_HEAD: usize = 64 * 1024; +/// Most header lines. +const MAX_HEADERS: usize = 256; +/// Longest chunk-size or trailer line. +const MAX_LINE: usize = 8 * 1024; + +/// A response's status line and headers. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Head { + pub status: u16, + pub reason: String, + /// Header names as sent; look them up with [`Head::header`]. + pub headers: Vec<(String, String)>, +} + +impl Head { + /// First value of a header (case-insensitive name). + pub fn header(&self, name: &str) -> Option<&str> { + self.headers.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str()) + } +} + +/// Read one line ending in LF (CR stripped), at most `max` bytes, one byte at a time (so nothing +/// past the line is consumed: a proxy tunnel's TLS bytes follow its head). +fn read_line(r: &mut impl Read, max: usize) -> Result { + let mut line = Vec::new(); + let mut b = [0u8; 1]; + loop { + let n = r.read(&mut b)?; + if n == 0 { + return Err(NetError::Protocol("connection closed in the middle of a line".into())); + } + if b[0] == b'\n' { + break; + } + if line.len() >= max { + return Err(NetError::Protocol("line too long".into())); + } + line.push(b[0]); + } + if line.last() == Some(&b'\r') { + line.pop(); + } + Ok(String::from_utf8_lossy(&line).into_owned()) +} + +/// Read a status line and headers. `1xx` interim responses are skipped. +pub fn read_head(r: &mut impl Read) -> Result { + loop { + let mut used = 0usize; + let status_line = read_line(r, MAX_HEAD)?; + used += status_line.len(); + let mut parts = status_line.splitn(3, ' '); + let version = parts.next().unwrap_or_default(); + if !version.starts_with("HTTP/1.") { + return Err(NetError::Protocol(format!("not an HTTP/1.x response: {:?}", status_line.chars().take(40).collect::()))); + } + let status: u16 = parts + .next() + .and_then(|s| s.parse().ok()) + .filter(|s| (100..1000).contains(s)) + .ok_or_else(|| NetError::Protocol("bad status code".into()))?; + let reason = parts.next().unwrap_or_default().to_string(); + let mut headers = Vec::new(); + loop { + let line = read_line(r, MAX_HEAD)?; + used += line.len() + 2; + if used > MAX_HEAD { + return Err(NetError::Protocol("response head larger than 64 KiB".into())); + } + if line.is_empty() { + break; + } + if headers.len() >= MAX_HEADERS { + return Err(NetError::Protocol("too many headers".into())); + } + let Some((k, v)) = line.split_once(':') else { + return Err(NetError::Protocol("malformed header line".into())); + }; + headers.push((k.trim().to_string(), v.trim().to_string())); + } + if (100..200).contains(&status) && status != 101 { + continue; + } + return Ok(Head { status, reason, headers }); + } +} + +/// How the body is delimited. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Framing { + Empty, + Length(u64), + Chunked, + Close, +} + +impl Framing { + pub(crate) fn of(head: &Head, request_was_head: bool) -> Result { + if request_was_head || head.status == 204 || head.status == 304 || (100..200).contains(&head.status) { + return Ok(Framing::Empty); + } + if head.header("transfer-encoding").is_some_and(|t| t.to_ascii_lowercase().contains("chunked")) { + return Ok(Framing::Chunked); + } + match head.header("content-length") { + Some(v) => v.trim().parse().map(Framing::Length).map_err(|_| NetError::Protocol("bad Content-Length".into())), + None => Ok(Framing::Close), + } + } +} + +/// Reads exactly the body, per its framing. +pub(crate) struct BodyReader { + inner: R, + state: State, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum State { + /// Bytes left (Content-Length, or of the current chunk). + Left { + left: u64, + chunked: bool, + }, + /// Before a chunk-size line (`first`: no CRLF of a previous chunk to consume). + ChunkStart { + first: bool, + }, + UntilClose, + Done, +} + +impl BodyReader { + pub(crate) fn new(inner: R, framing: Framing) -> Self { + let state = match framing { + Framing::Empty | Framing::Length(0) => State::Done, + Framing::Length(n) => State::Left { left: n, chunked: false }, + Framing::Chunked => State::ChunkStart { first: true }, + Framing::Close => State::UntilClose, + }; + BodyReader { inner, state } + } + + fn next_chunk(&mut self, first: bool) -> Result<(), NetError> { + if !first { + let crlf = read_line(&mut self.inner, 2)?; + if !crlf.is_empty() { + return Err(NetError::Protocol("missing CRLF after chunk".into())); + } + } + let line = read_line(&mut self.inner, MAX_LINE)?; + let size = line.split(';').next().unwrap_or_default().trim(); + let size = u64::from_str_radix(size, 16).map_err(|_| NetError::Protocol("bad chunk size".into()))?; + if size == 0 { + // trailers, then the empty line + let mut used = 0usize; + loop { + let t = read_line(&mut self.inner, MAX_LINE)?; + if t.is_empty() { + break; + } + used += t.len(); + if used > MAX_HEAD { + return Err(NetError::Protocol("trailers too large".into())); + } + } + self.state = State::Done; + } else { + self.state = State::Left { left: size, chunked: true }; + } + Ok(()) + } +} + +impl Read for BodyReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if buf.is_empty() { + return Ok(0); + } + loop { + match self.state { + State::Done => return Ok(0), + State::UntilClose => return self.inner.read(buf), + State::ChunkStart { first } => self.next_chunk(first)?, + State::Left { left, chunked } => { + let want = usize::try_from(left).unwrap_or(usize::MAX).min(buf.len()); + let n = self.inner.read(buf.get_mut(..want).unwrap_or_default())?; + if n == 0 { + return Err(NetError::Protocol("connection closed before the end of the body".into()).into()); + } + let left = left.saturating_sub(n as u64); + self.state = match (left, chunked) { + (0, true) => State::ChunkStart { first: false }, + (0, false) => State::Done, + (left, chunked) => State::Left { left, chunked }, + }; + return Ok(n); + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + fn body(raw: &[u8], framing: Framing) -> Result, NetError> { + let mut out = Vec::new(); + BodyReader::new(Cursor::new(raw.to_vec()), framing).read_to_end(&mut out)?; + Ok(out) + } + + #[test] + fn heads_and_bodies() { + let mut r = Cursor::new(b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 3\r\nX-A: b \r\n\r\nabc".to_vec()); + let h = read_head(&mut r).unwrap(); + assert_eq!((h.status, h.reason.as_str(), h.header("x-a")), (200, "OK", Some("b"))); + assert_eq!(Framing::of(&h, false).unwrap(), Framing::Length(3)); + assert_eq!(body(b"4;x=y\r\nWiki\r\n5\r\npedia\r\n0\r\nT: v\r\n\r\n", Framing::Chunked).unwrap(), b"Wikipedia"); + assert_eq!(body(b"rest", Framing::Close).unwrap(), b"rest"); + } + + #[test] + fn hostile_responses_are_errors() { + for raw in [&b"SSH-2.0\r\n\r\n"[..], b"HTTP/1.1 abc OK\r\n\r\n", b"HTTP/1.1 200 OK\r\nnocolon\r\n\r\n", b"HTTP/1.1 200"] { + assert!(read_head(&mut Cursor::new(raw.to_vec())).is_err()); + } + let huge = format!("HTTP/1.1 200 OK\r\n{}\r\n", "X: y\r\n".repeat(20_000)); + assert!(read_head(&mut Cursor::new(huge.into_bytes())).is_err()); + assert!(body(b"zz\r\n", Framing::Chunked).is_err()); + assert!(body(b"ffffffffffffffffffff\r\n", Framing::Chunked).is_err()); + assert!(body(b"3\r\nabcX", Framing::Chunked).is_err()); + assert!(body(b"ab", Framing::Length(5)).is_err()); + } +} diff --git a/crates/net/src/lib.rs b/crates/net/src/lib.rs new file mode 100644 index 0000000000..2c45833a75 --- /dev/null +++ b/crates/net/src/lib.rs @@ -0,0 +1,48 @@ +//! HTTP(S) for everything that talks to a server: Immich, publish services, map tiles and +//! geocoding, model downloads. Pure Rust: `std::net` sockets and rustls with the RustCrypto +//! provider (no `ring`/`aws-lc-rs`, nothing compiled from C), Mozilla roots from `webpki-roots`, +//! gzip through `flate2`'s Rust backend. +//! +//! - [`Client`] with [`ClientConfig`]: connect and stall timeouts, redirects (credentials never +//! follow a redirect to another origin, and https never downgrades to http), an optional HTTP +//! proxy (`CONNECT` for https), and per-connection [`Trust`]: extra CA certificates, pinned +//! fingerprints, and trust-on-first-use with a confirmation hook for self-signed home servers. +//! - [`Request`]: GET/HEAD/POST/PUT/PATCH/DELETE, headers ([`Request::secret_header`] / +//! [`Request::bearer`] for credentials), JSON or raw bodies, streaming [`Multipart`] uploads +//! with progress, cancellation through an `AtomicBool`. +//! - [`Response`]: status, headers, the body as a [`std::io::Read`] stream (gzip decoded), or +//! capped [`Response::bytes`] / [`Response::text`] / [`Response::json`]. +//! - [`Client::download`]: resumable, size-capped, SHA-256-verified downloads to a file. +//! +//! Blocking by design: callers run requests on worker threads. Logging never includes header +//! values, bodies or query strings (they carry API keys and share keys). +//! +//! Native only: on wasm32 this crate is empty. + +#![cfg(not(target_arch = "wasm32"))] +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +mod client; +mod conn; +mod download; +mod error; +pub mod http; +mod multipart; +mod tls; +mod url; + +pub use client::{Client, ClientConfig, Method, Progress, Request, Response}; +pub use download::{DownloadOptions, Downloaded}; +pub use error::NetError; +pub use multipart::Multipart; +pub use tls::{CertInfo, Fingerprint, TofuHook, Trust}; +pub use url::Url; + +/// `User-Agent`: the brand's binary name and the version. +pub fn user_agent() -> String { + format!("{}/{}", dac_brand::BINARY, env!("CARGO_PKG_VERSION")) +} + +#[cfg(test)] +mod tests; diff --git a/crates/net/src/multipart.rs b/crates/net/src/multipart.rs new file mode 100644 index 0000000000..18a6c3870d --- /dev/null +++ b/crates/net/src/multipart.rs @@ -0,0 +1,214 @@ +//! `multipart/form-data` bodies that stream: file parts are read from disk (or any reader) while +//! sending, never loaded whole, and the total length is known up front so the request carries a +//! `Content-Length` and progress has a denominator. + +use std::fs::File; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use sha2::{Digest, Sha256}; + +use crate::NetError; + +enum Data { + Bytes(Vec), + File(PathBuf, u64), + Reader(Box, u64), +} + +struct Part { + head: String, + data: Data, +} + +impl Part { + fn len(&self) -> u64 { + match &self.data { + Data::Bytes(b) => b.len() as u64, + Data::File(_, n) | Data::Reader(_, n) => *n, + } + } +} + +/// A `multipart/form-data` body under construction. +pub struct Multipart { + boundary: String, + parts: Vec, +} + +impl std::fmt::Debug for Multipart { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Multipart").field("parts", &self.parts.len()).field("len", &self.len()).finish() + } +} + +impl Default for Multipart { + fn default() -> Self { + Multipart::new() + } +} + +/// Quote a form field or file name: no line breaks, quotes escaped. +fn quoted(s: &str) -> String { + s.chars().filter(|c| *c != '\r' && *c != '\n').map(|c| if c == '"' { "%22".to_string() } else { c.to_string() }).collect() +} + +fn clean(s: &str) -> String { + s.chars().filter(|c| !c.is_control()).collect() +} + +impl Multipart { + pub fn new() -> Multipart { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let mut h = Sha256::new(); + h.update(format!("{:?}{}{:?}", std::time::SystemTime::now(), std::process::id(), std::thread::current().id())); + h.update(COUNTER.fetch_add(1, Ordering::Relaxed).to_le_bytes()); + let hex: String = h.finalize().iter().take(16).map(|b| format!("{b:02x}")).collect(); + Multipart { boundary: format!("----dac-{hex}"), parts: Vec::new() } + } + + fn push(&mut self, name: &str, file: Option<(&str, &str)>, data: Data) { + let mut head = format!("--{}\r\nContent-Disposition: form-data; name=\"{}\"", self.boundary, quoted(name)); + if let Some((filename, content_type)) = file { + head.push_str(&format!("; filename=\"{}\"\r\nContent-Type: {}", quoted(filename), clean(content_type))); + } + head.push_str("\r\n\r\n"); + self.parts.push(Part { head, data }); + } + + /// A text field. + pub fn text(mut self, name: &str, value: &str) -> Multipart { + self.push(name, None, Data::Bytes(value.as_bytes().to_vec())); + self + } + + /// A file part from memory. + pub fn bytes(mut self, name: &str, filename: &str, content_type: &str, data: Vec) -> Multipart { + self.push(name, Some((filename, content_type)), Data::Bytes(data)); + self + } + + /// A file part streamed from disk (its size is taken now; a file that changes size while + /// uploading fails the request rather than sending a corrupt body). + pub fn file(mut self, name: &str, filename: &str, content_type: &str, path: &Path) -> Result { + let len = std::fs::metadata(path).map_err(|e| NetError::Io(format!("{}: {e}", path.display())))?.len(); + self.push(name, Some((filename, content_type)), Data::File(path.to_path_buf(), len)); + Ok(self) + } + + /// A file part streamed from a reader that yields exactly `len` bytes. + pub fn reader(mut self, name: &str, filename: &str, content_type: &str, reader: Box, len: u64) -> Multipart { + self.push(name, Some((filename, content_type)), Data::Reader(reader, len)); + self + } + + /// The `Content-Type` header value. + pub fn content_type(&self) -> String { + format!("multipart/form-data; boundary={}", self.boundary) + } + + fn closing(&self) -> String { + format!("--{}--\r\n", self.boundary) + } + + /// Exact body length in bytes. + pub fn len(&self) -> u64 { + let parts: u64 = self.parts.iter().map(|p| (p.head.len() as u64).saturating_add(p.len()).saturating_add(2)).fold(0, u64::saturating_add); + parts.saturating_add(self.closing().len() as u64) + } + + pub fn is_empty(&self) -> bool { + self.parts.is_empty() + } + + /// Write the body, calling `progress(sent)` as it goes. + pub(crate) fn write_to(self, w: &mut impl Write, progress: &mut dyn FnMut(u64)) -> Result<(), NetError> { + let closing = self.closing(); + let mut sent = 0u64; + let mut buf = vec![0u8; 64 * 1024]; + for part in self.parts { + w.write_all(part.head.as_bytes())?; + sent = sent.saturating_add(part.head.len() as u64); + let expected = part.len(); + match part.data { + Data::Bytes(b) => { + for chunk in b.chunks(64 * 1024) { + w.write_all(chunk)?; + sent = sent.saturating_add(chunk.len() as u64); + progress(sent); + } + } + Data::File(path, _) => { + let f = File::open(&path).map_err(|e| NetError::Io(format!("{}: {e}", path.display())))?; + sent = copy_exact(f, expected, w, &mut buf, sent, progress)?; + } + Data::Reader(r, _) => sent = copy_exact(r, expected, w, &mut buf, sent, progress)?, + } + w.write_all(b"\r\n")?; + sent = sent.saturating_add(2); + } + w.write_all(closing.as_bytes())?; + progress(sent.saturating_add(closing.len() as u64)); + Ok(()) + } +} + +/// Copy exactly `len` bytes from `r` (fewer or more is an error: the Content-Length is promised). +fn copy_exact(mut r: impl Read, len: u64, w: &mut impl Write, buf: &mut [u8], mut sent: u64, progress: &mut dyn FnMut(u64)) -> Result { + let mut left = len; + while left > 0 { + let want = usize::try_from(left).unwrap_or(usize::MAX).min(buf.len()); + let chunk = buf.get_mut(..want).unwrap_or_default(); + let n = r.read(chunk).map_err(|e| NetError::Io(e.to_string()))?; + if n == 0 { + return Err(NetError::Io("upload source ended early (file changed while uploading?)".into())); + } + w.write_all(chunk.get(..n).unwrap_or_default())?; + left = left.saturating_sub(n as u64); + sent = sent.saturating_add(n as u64); + progress(sent); + } + let mut probe = [0u8; 1]; + if r.read(&mut probe).map_err(|e| NetError::Io(e.to_string()))? != 0 { + return Err(NetError::Io("upload source is longer than announced (file changed while uploading?)".into())); + } + Ok(sent) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn length_matches_bytes_written() { + let m = Multipart::new().text("deviceId", "x").bytes("a\"b", "f\r\n.jpg", "image/jpeg", vec![7; 100_000]).reader( + "r", + "r.bin", + "application/octet-stream", + Box::new(std::io::Cursor::new(vec![1u8; 10])), + 10, + ); + let len = m.len(); + let mut out = Vec::new(); + let mut last = 0; + m.write_to(&mut out, &mut |s| last = s).unwrap(); + assert_eq!(out.len() as u64, len); + assert_eq!(last, len); + let text = String::from_utf8_lossy(&out); + assert!(text.contains("name=\"a%22b\"; filename=\"f.jpg\"")); + } + + #[test] + fn short_or_long_reader_is_an_error() { + let m = Multipart::new().reader("r", "r", "x/y", Box::new(std::io::Cursor::new(vec![1u8; 5])), 10); + assert!(m.write_to(&mut Vec::new(), &mut |_| {}).is_err()); + let m = Multipart::new().reader("r", "r", "x/y", Box::new(std::io::Cursor::new(vec![1u8; 15])), 10); + assert!(m.write_to(&mut Vec::new(), &mut |_| {}).is_err()); + } + + #[test] + fn boundaries_differ() { + assert_ne!(Multipart::new().boundary, Multipart::new().boundary); + } +} diff --git a/crates/net/src/tests.rs b/crates/net/src/tests.rs new file mode 100644 index 0000000000..b9970020ec --- /dev/null +++ b/crates/net/src/tests.rs @@ -0,0 +1,441 @@ +//! End-to-end tests against in-process servers on 127.0.0.1 (no external network). + +use std::io::{BufRead, BufReader, Read, Write}; +use std::net::TcpListener; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::Duration; + +use super::*; + +#[derive(Debug, Clone)] +struct Req { + method: String, + target: String, + headers: Vec<(String, String)>, + body: Vec, +} + +impl Req { + fn header(&self, n: &str) -> Option<&str> { + self.headers.iter().find(|(k, _)| k.eq_ignore_ascii_case(n)).map(|(_, v)| v.as_str()) + } +} + +fn read_req(s: impl Read) -> Option { + let mut r = BufReader::new(s); + let mut line = String::new(); + r.read_line(&mut line).ok()?; + let mut it = line.split_whitespace(); + let (method, target) = (it.next()?.to_string(), it.next()?.to_string()); + let mut headers = Vec::new(); + loop { + let mut l = String::new(); + r.read_line(&mut l).ok()?; + let l = l.trim_end(); + if l.is_empty() { + break; + } + let (k, v) = l.split_once(':')?; + headers.push((k.trim().to_string(), v.trim().to_string())); + } + let len: usize = headers.iter().find(|(k, _)| k.eq_ignore_ascii_case("content-length")).and_then(|(_, v)| v.parse().ok()).unwrap_or(0); + let mut body = vec![0; len]; + r.read_exact(&mut body).ok()?; + Some(Req { method, target, headers, body }) +} + +fn response(status: &str, headers: &[(&str, &str)], body: &[u8]) -> Vec { + let mut out = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\n", body.len()); + for (k, v) in headers { + out.push_str(&format!("{k}: {v}\r\n")); + } + out.push_str("\r\n"); + let mut out = out.into_bytes(); + out.extend_from_slice(body); + out +} + +type Handler = Arc Vec + Send + Sync>; + +/// Serve `n` connections with `handler`; the requests seen are collected. +fn serve(n: usize, handler: impl Fn(Req) -> Vec + Send + Sync + 'static) -> (u16, Arc>>, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let seen = Arc::new(Mutex::new(Vec::new())); + let seen2 = seen.clone(); + let handler: Handler = Arc::new(handler); + let h = std::thread::spawn(move || { + for _ in 0..n { + let Ok((mut s, _)) = listener.accept() else { return }; + let Some(req) = read_req(&mut s) else { continue }; + seen2.lock().unwrap().push(req.clone()); + let _ = s.write_all(&handler(req)); + } + }); + (port, seen, h) +} + +fn client() -> Client { + Client::new(ClientConfig { stall_timeout: Duration::from_secs(5), ..ClientConfig::default() }).unwrap() +} + +#[test] +fn get_json_gzip_and_user_agent() { + let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + gz.write_all(br#"{"version":"1.2.3"}"#).unwrap(); + let gz = gz.finish().unwrap(); + let (port, seen, h) = serve(1, move |_| response("200 OK", &[("Content-Encoding", "gzip"), ("Content-Type", "application/json")], &gz)); + #[derive(serde::Deserialize)] + struct V { + version: String, + } + let v: V = client().get(&format!("http://127.0.0.1:{port}/api/server/version")).send().unwrap().error_for_status().unwrap().json().unwrap(); + assert_eq!(v.version, "1.2.3"); + h.join().unwrap(); + let req = seen.lock().unwrap()[0].clone(); + assert_eq!(req.header("user-agent"), Some(user_agent().as_str())); + assert!(user_agent().starts_with(dac_brand::BINARY)); + assert_eq!(req.header("accept-encoding"), Some("gzip")); +} + +#[test] +fn methods_and_bodies() { + let (port, seen, h) = serve(4, |r| response("200 OK", &[], &r.body)); + let c = client(); + let base = format!("http://127.0.0.1:{port}/x"); + assert_eq!(c.post(&base).json(&serde_json::json!({"a": 1})).send().unwrap().text().unwrap(), r#"{"a":1}"#); + assert_eq!(c.put(&base).body("text/plain", b"put".to_vec()).send().unwrap().bytes().unwrap(), b"put"); + assert_eq!(c.patch(&base).json(&[1, 2]).send().unwrap().text().unwrap(), "[1,2]"); + let r = c.delete(&base).send().unwrap(); + assert!(r.is_success()); + h.join().unwrap(); + let seen = seen.lock().unwrap(); + let methods: Vec<&str> = seen.iter().map(|r| r.method.as_str()).collect(); + assert_eq!(methods, ["POST", "PUT", "PATCH", "DELETE"]); + assert_eq!(seen[0].header("content-type"), Some("application/json")); +} + +#[test] +fn error_status_and_bad_headers() { + let (port, _, h) = serve(1, |_| response("404 Not Found", &[], b"nope")); + let e = client().get(&format!("http://127.0.0.1:{port}/")).send().unwrap().error_for_status().unwrap_err(); + assert_eq!(e, NetError::Status { code: 404, reason: "Not Found".into() }); + h.join().unwrap(); + let e = client().get("http://127.0.0.1:1/").header("X", "a\r\nInjected: 1").send().unwrap_err(); + assert!(matches!(e, NetError::BadHeader(_))); + assert!(!e.to_string().contains("Injected")); +} + +#[test] +fn multipart_upload_streams_with_progress() { + let dir = tempdir("mp"); + let file = dir.join("photo.jpg"); + std::fs::write(&file, vec![0xAB; 300_000]).unwrap(); + let (port, seen, h) = serve(1, |r| response("201 Created", &[], format!("{}", r.body.len()).as_bytes())); + let form = Multipart::new().text("deviceAssetId", "abc").file("assetData", "photo.jpg", "image/jpeg", &file).unwrap(); + let expected = form.len(); + let mut calls = 0u32; + let mut last = (0, None); + let mut progress = |s: u64, t: Option| { + calls += 1; + last = (s, t); + }; + let r = client().post(&format!("http://127.0.0.1:{port}/api/assets")).multipart(form).progress(&mut progress).send().unwrap(); + assert_eq!(r.status, 201); + assert_eq!(r.text().unwrap(), expected.to_string()); + assert!(calls > 1); + assert_eq!(last, (expected, Some(expected))); + h.join().unwrap(); + let req = seen.lock().unwrap()[0].clone(); + assert!(req.header("content-type").unwrap().starts_with("multipart/form-data; boundary=")); + let body = String::from_utf8_lossy(&req.body); + assert!(body.contains("name=\"assetData\"; filename=\"photo.jpg\"")); + let _ = std::fs::remove_dir_all(dir); +} + +#[test] +fn redirects_keep_credentials_on_the_origin_only() { + let (other, other_seen, h2) = serve(1, |_| response("200 OK", &[], b"elsewhere")); + let (port, seen, h1) = serve(2, move |r| { + if r.target == "/a" { + response("302 Found", &[("Location", "/b")], b"") + } else { + response("307 Temporary Redirect", &[("Location", &format!("http://localhost:{other}/c"))], b"") + } + }); + let r = client().post(&format!("http://127.0.0.1:{port}/a")).bearer("s3cret-token").header("X-Api-Key", "k3y").json(&1).send().unwrap(); + assert_eq!(r.text().unwrap(), "elsewhere"); + h1.join().unwrap(); + h2.join().unwrap(); + let seen = seen.lock().unwrap(); + assert_eq!(seen[1].method, "GET"); // 302 after POST becomes GET + assert!(seen[1].header("authorization").is_some()); + let other_seen = other_seen.lock().unwrap(); + assert_eq!(other_seen[0].target, "/c"); + assert!(other_seen[0].header("authorization").is_none()); + assert!(other_seen[0].header("x-api-key").is_none()); +} + +#[test] +fn too_many_redirects() { + let (port, _, h) = serve(3, |_| response("302 Found", &[("Location", "/loop")], b"")); + let c = Client::new(ClientConfig { max_redirects: 2, ..ClientConfig::default() }).unwrap(); + assert!(matches!(c.get(&format!("http://127.0.0.1:{port}/")).send(), Err(NetError::Redirect(_)))); + h.join().unwrap(); +} + +#[test] +fn stall_timeout_and_cancel() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let h = std::thread::spawn(move || { + let mut held = Vec::new(); + for _ in 0..2 { + if let Ok((s, _)) = listener.accept() { + held.push(s); // never answer + } + } + std::thread::sleep(Duration::from_secs(3)); + }); + let c = Client::new(ClientConfig { stall_timeout: Duration::from_millis(600), ..ClientConfig::default() }).unwrap(); + let t = std::time::Instant::now(); + assert_eq!(c.get(&format!("http://127.0.0.1:{port}/")).send().unwrap_err(), NetError::Stalled); + assert!(t.elapsed() < Duration::from_secs(3)); + + let c = Client::new(ClientConfig { stall_timeout: Duration::from_secs(60), ..ClientConfig::default() }).unwrap(); + let flag = Arc::new(AtomicBool::new(false)); + let f2 = flag.clone(); + std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(300)); + f2.store(true, Ordering::Relaxed); + }); + let t = std::time::Instant::now(); + assert_eq!(c.get(&format!("http://127.0.0.1:{port}/")).cancel(flag).send().unwrap_err(), NetError::Cancelled); + assert!(t.elapsed() < Duration::from_secs(5)); + h.join().unwrap(); +} + +#[test] +fn connect_failure_is_an_error() { + let port = TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); // closed again + assert!(matches!(client().get(&format!("http://127.0.0.1:{port}/")).send(), Err(NetError::Connect(_)))); +} + +#[test] +fn http_proxy_gets_absolute_form() { + let (port, seen, h) = serve(1, |_| response("200 OK", &[], b"via proxy")); + let c = Client::new(ClientConfig { proxy: Some(format!("http://127.0.0.1:{port}")), ..ClientConfig::default() }).unwrap(); + assert_eq!(c.get("http://photos.invalid:2283/api/x?k=v").send().unwrap().text().unwrap(), "via proxy"); + h.join().unwrap(); + assert_eq!(seen.lock().unwrap()[0].target, "http://photos.invalid:2283/api/x?k=v"); + assert!(Client::new(ClientConfig { proxy: Some("https://p".into()), ..ClientConfig::default() }).is_err()); +} + +#[test] +fn chunked_and_truncated_bodies() { + let (port, _, h) = serve(2, |r| { + if r.target == "/chunked" { + b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n2\r\nde\r\n0\r\n\r\n".to_vec() + } else { + b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nshort".to_vec() + } + }); + assert_eq!(client().get(&format!("http://127.0.0.1:{port}/chunked")).send().unwrap().text().unwrap(), "abcde"); + assert!(client().get(&format!("http://127.0.0.1:{port}/short")).send().unwrap().bytes().is_err()); + h.join().unwrap(); +} + +#[test] +fn body_cap() { + let (port, _, h) = serve(1, |_| response("200 OK", &[], &[1u8; 5000])); + let c = Client::new(ClientConfig { max_body: 1000, ..ClientConfig::default() }).unwrap(); + assert_eq!(c.get(&format!("http://127.0.0.1:{port}/")).send().unwrap().bytes().unwrap_err(), NetError::TooLarge(1000)); + h.join().unwrap(); +} + +fn tempdir(tag: &str) -> std::path::PathBuf { + let d = std::env::temp_dir().join(format!("dac-net-test-{tag}-{}-{:?}", std::process::id(), std::thread::current().id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +fn sha256_hex(data: &[u8]) -> String { + use sha2::Digest; + sha2::Sha256::digest(data).iter().map(|b| format!("{b:02x}")).collect() +} + +fn range_server(data: Vec, n: usize) -> (u16, Arc>>, JoinHandle<()>) { + serve(n, move |r| { + match r.header("range").and_then(|v| v.strip_prefix("bytes=")).and_then(|v| v.strip_suffix('-')).and_then(|v| v.parse::().ok()) { + Some(start) if start < data.len() => { + let cr = format!("bytes {start}-{}/{}", data.len() - 1, data.len()); + response("206 Partial Content", &[("Content-Range", &cr)], &data[start..]) + } + _ => response("200 OK", &[], &data), + } + }) +} + +#[test] +fn download_resumes_and_verifies() { + let data: Vec = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect(); + let sha = sha256_hex(&data); + let dir = tempdir("dl"); + let dest = dir.join("model.bin"); + std::fs::write(dir.join("model.bin.part"), &data[..80_000]).unwrap(); + let (port, seen, h) = range_server(data.clone(), 1); + let mut last = 0; + let mut progress = |s: u64, _t: Option| last = s; + let opts = + DownloadOptions { size: Some(data.len() as u64), sha256: Some(sha.to_uppercase()), progress: Some(&mut progress), ..Default::default() }; + let got = client().download(&format!("http://127.0.0.1:{port}/m"), &dest, opts).unwrap(); + h.join().unwrap(); + assert_eq!(got, Downloaded { bytes: data.len() as u64, sha256: sha.clone() }); + assert_eq!(last, data.len() as u64); + assert_eq!(std::fs::read(&dest).unwrap(), data); + assert!(!dir.join("model.bin.part").exists()); + let seen = seen.lock().unwrap(); + assert_eq!(seen[0].header("range"), Some("bytes=80000-")); + assert_eq!(seen[0].header("accept-encoding"), Some("identity")); + + // a wrong hash deletes the part file and keeps no file under the real name + let dest2 = dir.join("bad.bin"); + let (port, _, h) = range_server(data.clone(), 1); + let opts = DownloadOptions { sha256: Some("00".repeat(32)), ..Default::default() }; + assert!(matches!(client().download(&format!("http://127.0.0.1:{port}/m"), &dest2, opts), Err(NetError::Verify(_)))); + h.join().unwrap(); + assert!(!dest2.exists() && !dir.join("bad.bin.part").exists()); + + // a cap smaller than the file + let (port, _, h) = range_server(data, 1); + let opts = DownloadOptions { max_size: 1000, ..Default::default() }; + assert!(matches!(client().download(&format!("http://127.0.0.1:{port}/m"), &dir.join("big.bin"), opts), Err(NetError::TooLarge(_)))); + h.join().unwrap(); + let _ = std::fs::remove_dir_all(dir); +} + +// --- TLS ------------------------------------------------------------------------------------- + +/// A test CA and a `localhost`/127.0.0.1 leaf it signed (P-256, valid until 2126), generated with +/// openssl for these tests only. +const CA_B64: &str = "MIIBjDCCATGgAwIBAgIUabPpTeBrc750uF10VHKviz4AX94wCgYIKoZIzj0EAwIwEjEQMA4GA1UEAwwHVGVzdCBDQTAgFw0yNjEwMTAwMDA1NDlaGA8yMTI2MDkxNjAwMDU0OVowEjEQMA4GA1UEAwwHVGVzdCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABB+i1UqTwg54S6Ef3QlGtBCju6tkmrJoeDPLw4AYwvfr07fhEP2asbQew8dGE8EspL4KyYwnTtsBEcocsbAah4ujYzBhMB0GA1UdDgQWBBSqjOvGOiSW/kbE/YVt+Q3Qlm7bEzAfBgNVHSMEGDAWgBSqjOvGOiSW/kbE/YVt+Q3Qlm7bEzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwICBDAKBggqhkjOPQQDAgNJADBGAiEAzHuuueM9+8Laj3rzm9LfkZuBS5cDoVwxlYPlHsfUvmYCIQDGh047dSWnC7wIfN+bryPuQ7wg023H9UdDlMBMsHDV+w=="; +const LEAF_B64: &str = "MIIBqTCCAU6gAwIBAgIUHPx6OQjNVD01xDDk464Uj1xwmzcwCgYIKoZIzj0EAwIwEjEQMA4GA1UEAwwHVGVzdCBDQTAgFw0yNjEwMTAwMDA1NDlaGA8yMTI2MDkxNjAwMDU0OVowFDESMBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFPSyEHSnzWNFWHjo6Fsccp8IUISLOIv2JcW4dwim6VlrvmcgH4fSRtMLq9JDbrs7eGp5J/7qj5VL3iqt4kGTI6N+MHwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMAkGA1UdEwQCMAAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHQYDVR0OBBYEFIkl10ciEZxIBMwplQXH2M/FHL6EMB8GA1UdIwQYMBaAFKqM68Y6JJb+RsT9hW35DdCWbtsTMAoGCCqGSM49BAMCA0kAMEYCIQCGcYFLw3UXrn8ea/Hfn4plLzgTQLoQzYhaA86WzbiexQIhAM/ozbpH7BFWxXjrbJfUwlFZ5A6k670RBOEwjjudfnO4"; +const KEY_B64: &str = "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgP7on22FsY1CzXydImw0X41brmtJ52CZx4n9+5T6HDV+hRANCAAQU9LIQdKfNY0VYeOjoWxxynwhQhIs4i/Ylxbh3CKbpWWu+ZyAfh9JG0wur0kNuuzt4ankn/uqPlUveKq3iQZMj"; + +fn b64(s: &str) -> Vec { + tls::base64_decode(s).unwrap() +} + +/// An HTTPS server for `n` connections answering "secure". +fn tls_serve(n: usize) -> (u16, JoinHandle<()>) { + use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer}; + let config = rustls::ServerConfig::builder_with_provider(Arc::new(rustls_rustcrypto::provider())) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert(vec![CertificateDer::from(b64(LEAF_B64))], PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(b64(KEY_B64)))) + .unwrap(); + let config = Arc::new(config); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let port = listener.local_addr().unwrap().port(); + let h = std::thread::spawn(move || { + for _ in 0..n { + let Ok((s, _)) = listener.accept() else { return }; + s.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let conn = rustls::ServerConnection::new(config.clone()).unwrap(); + let mut tls = rustls::StreamOwned::new(conn, s); + if read_req(&mut tls).is_some() { + let _ = tls.write_all(&response("200 OK", &[], b"secure")); + tls.conn.send_close_notify(); + let _ = tls.flush(); + } + } + }); + (port, h) +} + +fn get_tls(trust: Trust, port: u16) -> Result { + let c = Client::new(ClientConfig { trust, stall_timeout: Duration::from_secs(5), ..ClientConfig::default() })?; + c.get(&format!("https://localhost:{port}/")).send()?.text() +} + +#[test] +fn tls_trust_ca_pin_and_tofu() { + let leaf_fp = Fingerprint::of(&b64(LEAF_B64)); + let (port, h) = tls_serve(5); + + // untrusted: the error names the fingerprint + match get_tls(Trust::new(), port) { + Err(NetError::UntrustedCertificate { host, fingerprint }) => { + assert_eq!(host, "localhost"); + assert_eq!(fingerprint, leaf_fp.to_string()); + } + other => panic!("{other:?}"), + } + // the connection's own CA + assert_eq!(get_tls(Trust::new().with_ca(&b64(CA_B64)).unwrap(), port).unwrap(), "secure"); + // a pinned fingerprint + assert_eq!(get_tls(Trust::new().with_pinned(leaf_fp), port).unwrap(), "secure"); + // trust on first use: declined, then accepted (and pinned) + assert!(matches!(get_tls(Trust::new().with_tofu(Arc::new(|_| false)), port), Err(NetError::UntrustedCertificate { .. }))); + let seen = Arc::new(Mutex::new(None)); + let s2 = seen.clone(); + let trust = Trust::new().with_tofu(Arc::new(move |info: &CertInfo| { + *s2.lock().unwrap() = Some(info.fingerprint); + true + })); + assert_eq!(get_tls(trust.clone(), port).unwrap(), "secure"); + assert_eq!(*seen.lock().unwrap(), Some(leaf_fp)); + assert_eq!(trust.pinned(), vec![leaf_fp]); + h.join().unwrap(); +} + +#[test] +fn ca_pem_is_accepted() { + let pem = format!("-----BEGIN CERTIFICATE-----\n{}\n-----END CERTIFICATE-----\n", CA_B64); + assert!(Trust::new().with_ca(pem.as_bytes()).is_ok()); +} + +// --- logs never carry secrets ----------------------------------------------------------------- + +struct Capture; +static LOGS: Mutex = Mutex::new(String::new()); + +impl log::Log for Capture { + fn enabled(&self, _: &log::Metadata) -> bool { + true + } + fn log(&self, r: &log::Record) { + LOGS.lock().unwrap().push_str(&format!("{} {}\n", r.target(), r.args())); + } + fn flush(&self) {} +} + +#[test] +fn secrets_never_reach_the_log_or_debug_output() { + static ONCE: std::sync::Once = std::sync::Once::new(); + ONCE.call_once(|| { + let _ = log::set_logger(&Capture); + log::set_max_level(log::LevelFilter::Trace); + }); + let (port, _, h) = serve(1, |_| response("401 Unauthorized", &[("Set-Cookie", "session=c00kie-secret")], b"")); + let r = client() + .get(&format!("http://127.0.0.1:{port}/api/x?key=share-secret")) + .bearer("bearer-secret") + .header("x-api-key", "apikey-secret") + .send() + .unwrap(); + let debug = format!("{r:?}"); + let err = r.error_for_status().unwrap_err().to_string(); + h.join().unwrap(); + let logs = LOGS.lock().unwrap().clone(); + assert!(logs.contains("net: GET"), "logging works: {logs}"); + for text in [&logs, &debug, &err] { + for secret in ["bearer-secret", "apikey-secret", "share-secret", "c00kie-secret"] { + assert!(!text.contains(secret), "{secret} leaked into {text}"); + } + } +} diff --git a/crates/net/src/tls.rs b/crates/net/src/tls.rs new file mode 100644 index 0000000000..a57474ba83 --- /dev/null +++ b/crates/net/src/tls.rs @@ -0,0 +1,281 @@ +//! Which server certificates a connection trusts. +//! +//! A certificate passes when it chains to the Mozilla roots (`webpki-roots`) or to one of the +//! connection's extra CA certificates, or when the SHA-256 fingerprint of the server's own +//! certificate is pinned. Otherwise, when a trust-on-first-use hook is set, the hook sees the +//! certificate (host, fingerprint, why it failed) and decides; an accepted fingerprint is pinned +//! for the rest of the [`Trust`]'s life and is readable through [`Trust::pinned`] so the caller +//! can store it with the account. Home servers with self-signed certificates (common for Immich) +//! work this way without ever turning verification off: handshake signatures are always checked. + +use std::sync::{Arc, Mutex, PoisonError}; + +use rustls::client::WebPkiServerVerifier; +use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier}; +use rustls::crypto::CryptoProvider; +use rustls::pki_types::{CertificateDer, ServerName, UnixTime}; +use rustls::{DigitallySignedStruct, RootCertStore, SignatureScheme}; +use sha2::{Digest, Sha256}; + +use crate::NetError; + +/// SHA-256 of a DER certificate. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct Fingerprint(pub [u8; 32]); + +impl Fingerprint { + pub fn of(der: &[u8]) -> Fingerprint { + Fingerprint(Sha256::digest(der).into()) + } + + /// Parse the `AB:CD:…` form (colons and case optional). + pub fn parse(s: &str) -> Option { + let hex: String = s.chars().filter(|c| *c != ':').collect(); + if hex.len() != 64 { + return None; + } + let mut out = [0u8; 32]; + for (i, b) in out.iter_mut().enumerate() { + *b = u8::from_str_radix(hex.get(i * 2..i * 2 + 2)?, 16).ok()?; + } + Some(Fingerprint(out)) + } +} + +impl std::fmt::Display for Fingerprint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + for (i, b) in self.0.iter().enumerate() { + if i > 0 { + f.write_str(":")?; + } + write!(f, "{b:02X}")?; + } + Ok(()) + } +} + +impl std::fmt::Debug for Fingerprint { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Fingerprint({self})") + } +} + +/// What the trust-on-first-use hook is shown. +#[derive(Debug, Clone)] +pub struct CertInfo { + pub host: String, + pub fingerprint: Fingerprint, + /// Why normal verification refused it (self-signed, unknown issuer, expired, wrong name…). + pub reason: String, + /// The server's certificate (DER), for a details view. + pub der: Vec, +} + +/// Decides about a certificate nothing else vouches for: `true` accepts (and pins) it. +/// Called on the requesting thread during the handshake; a UI typically answers from a +/// fingerprint the user confirmed earlier, or fails the request and asks. +pub type TofuHook = Arc bool + Send + Sync>; + +/// The trust settings of one connection (account). Cheap to clone; clones share the pins. +#[derive(Clone, Default)] +pub struct Trust { + extra_roots: Vec>, + pinned: Arc>>, + tofu: Option, +} + +impl std::fmt::Debug for Trust { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Trust") + .field("extra_roots", &self.extra_roots.len()) + .field("pinned", &self.pinned()) + .field("tofu", &self.tofu.is_some()) + .finish() + } +} + +impl Trust { + /// Mozilla roots only. + pub fn new() -> Trust { + Trust::default() + } + + /// Also trust certificates issued by this CA (DER or PEM). + pub fn with_ca(mut self, cert: &[u8]) -> Result { + let ders = if cert.starts_with(b"-----") { pem_certificates(cert)? } else { vec![cert.to_vec()] }; + if ders.is_empty() { + return Err(NetError::Tls("no certificate in the CA file".into())); + } + self.extra_roots.extend(ders); + Ok(self) + } + + /// Trust the server certificate with this fingerprint (one the user confirmed earlier). + pub fn with_pinned(self, fp: Fingerprint) -> Trust { + self.pin(fp); + self + } + + /// Ask `hook` about certificates nothing else vouches for. + pub fn with_tofu(mut self, hook: TofuHook) -> Trust { + self.tofu = Some(hook); + self + } + + /// The pinned fingerprints, including those accepted by the hook since. + pub fn pinned(&self) -> Vec { + self.pinned.lock().unwrap_or_else(PoisonError::into_inner).clone() + } + + fn pin(&self, fp: Fingerprint) { + let mut p = self.pinned.lock().unwrap_or_else(PoisonError::into_inner); + if !p.contains(&fp) { + p.push(fp); + } + } + + fn is_pinned(&self, fp: &Fingerprint) -> bool { + self.pinned.lock().unwrap_or_else(PoisonError::into_inner).contains(fp) + } + + /// The rustls client configuration for this trust. + pub(crate) fn client_config(&self) -> Result, NetError> { + let provider = Arc::new(rustls_rustcrypto::provider()); + let mut roots = RootCertStore { roots: webpki_roots::TLS_SERVER_ROOTS.to_vec() }; + for der in &self.extra_roots { + roots.add(CertificateDer::from(der.clone())).map_err(|e| NetError::Tls(format!("CA certificate: {e}")))?; + } + let webpki = + WebPkiServerVerifier::builder_with_provider(Arc::new(roots), provider.clone()).build().map_err(|e| NetError::Tls(e.to_string()))?; + let verifier = Arc::new(Verifier { webpki, provider: provider.clone(), trust: self.clone() }); + let config = rustls::ClientConfig::builder_with_provider(provider) + .with_safe_default_protocol_versions() + .map_err(|e| NetError::Tls(e.to_string()))? + .dangerous() + .with_custom_certificate_verifier(verifier) + .with_no_client_auth(); + Ok(Arc::new(config)) + } +} + +/// Pull the CERTIFICATE blocks out of a PEM file. +fn pem_certificates(pem: &[u8]) -> Result>, NetError> { + let text = std::str::from_utf8(pem).map_err(|_| NetError::Tls("CA file is not text".into()))?; + let mut out = Vec::new(); + let mut rest = text; + while let Some(start) = rest.find("-----BEGIN CERTIFICATE-----") { + let body = rest.get(start + 27..).unwrap_or_default(); + let end = body.find("-----END CERTIFICATE-----").ok_or_else(|| NetError::Tls("unterminated PEM certificate".into()))?; + let b64: String = body.get(..end).unwrap_or_default().chars().filter(|c| !c.is_whitespace()).collect(); + out.push(base64_decode(&b64).ok_or_else(|| NetError::Tls("bad base64 in PEM certificate".into()))?); + rest = body.get(end..).unwrap_or_default(); + } + Ok(out) +} + +pub(crate) fn base64_decode(s: &str) -> Option> { + let mut out = Vec::with_capacity(s.len() / 4 * 3); + let (mut acc, mut bits) = (0u32, 0u32); + for c in s.bytes() { + let v = match c { + b'A'..=b'Z' => c - b'A', + b'a'..=b'z' => c - b'a' + 26, + b'0'..=b'9' => c - b'0' + 52, + b'+' => 62, + b'/' => 63, + b'=' => break, + _ => return None, + }; + acc = (acc << 6) | u32::from(v); + bits += 6; + if bits >= 8 { + bits -= 8; + out.push((acc >> bits) as u8); + acc &= (1 << bits) - 1; + } + } + Some(out) +} + +#[derive(Debug)] +struct Verifier { + webpki: Arc, + provider: Arc, + trust: Trust, +} + +impl ServerCertVerifier for Verifier { + fn verify_server_cert( + &self, + end_entity: &CertificateDer<'_>, + intermediates: &[CertificateDer<'_>], + server_name: &ServerName<'_>, + ocsp: &[u8], + now: UnixTime, + ) -> Result { + let reason = match self.webpki.verify_server_cert(end_entity, intermediates, server_name, ocsp, now) { + Ok(ok) => return Ok(ok), + Err(e) => e.to_string(), + }; + let fp = Fingerprint::of(end_entity.as_ref()); + if self.trust.is_pinned(&fp) { + return Ok(ServerCertVerified::assertion()); + } + let host = server_name.to_str().into_owned(); + if let Some(hook) = &self.trust.tofu { + let info = CertInfo { host: host.clone(), fingerprint: fp, reason, der: end_entity.as_ref().to_vec() }; + if hook(&info) { + log::info!("net: accepted certificate {fp} for {host}"); + self.trust.pin(fp); + return Ok(ServerCertVerified::assertion()); + } + } + Err(rustls::Error::Other(rustls::OtherError(Arc::new(NetError::UntrustedCertificate { host, fingerprint: fp.to_string() })))) + } + + fn verify_tls12_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls12_signature(message, cert, dss, &self.provider.signature_verification_algorithms) + } + + fn verify_tls13_signature( + &self, + message: &[u8], + cert: &CertificateDer<'_>, + dss: &DigitallySignedStruct, + ) -> Result { + rustls::crypto::verify_tls13_signature(message, cert, dss, &self.provider.signature_verification_algorithms) + } + + fn supported_verify_schemes(&self) -> Vec { + self.provider.signature_verification_algorithms.supported_schemes() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn fingerprint_round_trip() { + let fp = Fingerprint::of(b"abc"); + let s = fp.to_string(); + assert_eq!(s.len(), 95); + assert_eq!(Fingerprint::parse(&s), Some(fp)); + assert_eq!(Fingerprint::parse(&s.replace(':', "").to_lowercase()), Some(fp)); + assert_eq!(Fingerprint::parse("zz"), None); + assert_eq!(Fingerprint::parse(&"é".repeat(32)), None); + } + + #[test] + fn base64() { + assert_eq!(base64_decode("aGVsbG8=").unwrap(), b"hello"); + assert_eq!(base64_decode("!!"), None); + assert!(Trust::new().with_ca(b"-----BEGIN CERTIFICATE-----\nAAA").is_err()); + assert!(Trust::new().with_ca(b"-----nothing").is_err()); + } +} diff --git a/crates/net/src/url.rs b/crates/net/src/url.rs new file mode 100644 index 0000000000..ca49a2aea2 --- /dev/null +++ b/crates/net/src/url.rs @@ -0,0 +1,154 @@ +//! `http://` and `https://` URLs: just enough parsing to connect, send a request line and follow +//! redirects. Hostile input never panics; anything odd is a [`NetError::BadUrl`]. + +use crate::NetError; + +/// Longest URL accepted. +const MAX_URL: usize = 16 * 1024; + +/// A parsed `http://` or `https://` URL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Url { + /// `https://`. + pub tls: bool, + /// Host name or IP address (an IPv6 address without brackets). + pub host: String, + pub port: u16, + /// Path and query, starting with `/`. + pub path: String, +} + +impl Url { + pub fn parse(s: &str) -> Result { + let bad = |why: &str| NetError::BadUrl(why.to_string()); + if s.len() > MAX_URL { + return Err(bad("longer than 16 KiB")); + } + let s = s.trim(); + if s.chars().any(|c| c.is_control() || c == ' ') { + return Err(bad("contains spaces or control characters")); + } + let lower = s.get(..8).unwrap_or(s).to_ascii_lowercase(); + let (tls, rest) = if lower.starts_with("https://") { + (true, s.get(8..).unwrap_or_default()) + } else if lower.starts_with("http://") { + (false, s.get(7..).unwrap_or_default()) + } else { + return Err(bad("only http:// and https:// are supported")); + }; + let rest = rest.split('#').next().unwrap_or_default(); + let (authority, path) = match rest.find(['/', '?']) { + Some(i) => (rest.get(..i).unwrap_or_default(), rest.get(i..).unwrap_or_default()), + None => (rest, ""), + }; + if authority.contains('@') { + return Err(bad("credentials in URLs are not supported")); + } + let default_port = if tls { 443 } else { 80 }; + let (host, port) = if let Some(v6) = authority.strip_prefix('[') { + let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?; + let host = v6.get(..end).unwrap_or_default(); + let after = v6.get(end + 1..).unwrap_or_default(); + let port = match after.strip_prefix(':') { + Some(p) => parse_port(p)?, + None if after.is_empty() => default_port, + None => return Err(bad("text after the IPv6 address")), + }; + (host.to_string(), port) + } else { + match authority.rsplit_once(':') { + Some((h, p)) => (h.to_string(), parse_port(p)?), + None => (authority.to_string(), default_port), + } + }; + if host.is_empty() { + return Err(bad("no host")); + } + if !host.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | ':')) { + return Err(bad("host has unsupported characters (use the punycode form)")); + } + let path = if path.starts_with('?') { + format!("/{path}") + } else if path.is_empty() { + "/".to_string() + } else { + path.to_string() + }; + Ok(Url { tls, host: host.to_ascii_lowercase(), port, path }) + } + + /// The `Host` header value (port only when not the default). + pub fn host_header(&self) -> String { + let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() }; + if self.port == self.default_port() { host } else { format!("{host}:{}", self.port) } + } + + fn default_port(&self) -> u16 { + if self.tls { 443 } else { 80 } + } + + /// `scheme://host[:port]`, to compare origins (credentials never cross origins). + pub fn origin(&self) -> String { + format!("{}://{}", if self.tls { "https" } else { "http" }, self.host_header()) + } + + /// Resolve a `Location` header against this URL. + pub fn join(&self, location: &str) -> Result { + let l = location.trim(); + let lower = l.get(..8).unwrap_or(l).to_ascii_lowercase(); + if lower.starts_with("http://") || lower.starts_with("https://") { + return Url::parse(l); + } + if let Some(rest) = l.strip_prefix("//") { + return Url::parse(&format!("{}://{rest}", if self.tls { "https" } else { "http" })); + } + let path = if l.starts_with('/') { + l.to_string() + } else { + let base = self.path.split('?').next().unwrap_or("/"); + let dir = base.rsplit_once('/').map(|(d, _)| d).unwrap_or(""); + format!("{dir}/{l}") + }; + Url::parse(&format!("{}{path}", self.origin())) + } +} + +impl std::fmt::Display for Url { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}{}", self.origin(), self.path) + } +} + +fn parse_port(p: &str) -> Result { + match p.parse::() { + Ok(n) if n > 0 => Ok(n), + _ => Err(NetError::BadUrl(format!("bad port {p:?}"))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_and_joins() { + let u = Url::parse("HTTPS://Photos.Example:8443/api/x?y=1#frag").unwrap(); + assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "photos.example", 8443, "/api/x?y=1")); + assert_eq!(u.host_header(), "photos.example:8443"); + assert_eq!(u.join("/b").unwrap().to_string(), "https://photos.example:8443/b"); + assert_eq!(u.join("c?d").unwrap().path, "/api/c?d"); + assert_eq!(u.join("//other/z").unwrap().to_string(), "https://other/z"); + let v6 = Url::parse("http://[::1]:2283").unwrap(); + assert_eq!((v6.host.as_str(), v6.port, v6.path.as_str()), ("::1", 2283, "/")); + assert_eq!(v6.host_header(), "[::1]:2283"); + assert_eq!(Url::parse("http://h?q").unwrap().path, "/?q"); + } + + #[test] + fn hostile_urls_are_errors() { + for s in ["ftp://x", "http://", "http://a b", "http://u:p@h/", "http://h:0", "http://h:99999", "http://[::1", "http://h\r\n/", "http://é/"] { + assert!(Url::parse(s).is_err(), "{s}"); + } + assert!(Url::parse(&"h".repeat(20_000)).is_err()); + } +} diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index 13721b93e9..dac70bd3d7 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -40,6 +40,9 @@ pub const TABLE: &[(&str, Class)] = &[ ("tiff", Class::Layer(0)), ("sysmem", Class::Layer(0)), ("fetch", Class::Layer(0)), + ("net", Class::Layer(0)), + ("credentials", Class::Layer(0)), + ("hash", Class::Layer(0)), ("denoise-core", Class::Layer(1)), ("denoise", Class::Layer(1)), ("heif", Class::Layer(0)), @@ -82,6 +85,8 @@ pub const INTRA_LAYER_ORDER: &[&[&str]] = &[ &["brand", "tiff"], &["brand", "sysmem"], &["brand", "fetch"], + &["brand", "net"], + &["brand", "credentials"], &["brand", "heif"], &["geom", "color", "raster"], &["tiff", "raster"],