aleleba/lightcraft
1
0
Fork 0
mirror of https://github.com/storytold/lightcraft.git synced 2026-11-04 15:40:04 -06:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'worktree-agent-a783b26a16b464f44'

Browse Source
This commit is contained in:
Andrew Frahn
2026-10-10 11:29:54 +11:00
parent 4807cecb1a 5fe7835d8e
commit a9cef6548c
22 changed files with 3228 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+80
View File
@@ -310,6 +310,18 @@ dependencies = [
"syn 2.0.119",
]
[[package]]
name = "argon2"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3c3610892ee6e0cbce8ae2700349fcf8f98adb0dbfbee85aec3c9179d29cc072"
dependencies = [
"base64ct",
"blake2",
"cpufeatures",
"password-hash",
]
[[package]]
name = "arrayvec"
version = "0.7.8"
@@ -641,6 +653,15 @@ dependencies = [
"no_std_io2",
]
[[package]]
name = "blake2"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe"
dependencies = [
"digest",
]
[[package]]
name = "block"
version = "0.1.6"
@@ -1268,6 +1289,21 @@ dependencies = [
"serde",
]
[[package]]
name = "dac-credentials"
version = "0.4.0"
dependencies = [
"argon2",
"chacha20poly1305",
"dac-brand",
"getrandom 0.3.4",
"log",
"serde",
"serde_json",
"zbus",
"zeroize",
]
[[package]]
name = "dac-denoise"
version = "0.4.0"
@@ -1472,6 +1508,13 @@ dependencies = [
"wgpu",
]
[[package]]
name = "dac-hash"
version = "0.4.0"
dependencies = [
"sha1",
]
[[package]]
name = "dac-heif"
version = "0.4.0"
@@ -1522,6 +1565,21 @@ dependencies = [
"serde_json",
]
[[package]]
name = "dac-net"
version = "0.4.0"
dependencies = [
"dac-brand",
"flate2",
"log",
"rustls",
"rustls-rustcrypto",
"serde",
"serde_json",
"sha2",
"webpki-roots",
]
[[package]]
name = "dac-pipeline"
version = "0.4.0"
@@ -4353,6 +4411,17 @@ dependencies = [
"windows-link",
]
[[package]]
name = "password-hash"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166"
dependencies = [
"base64ct",
"rand_core 0.6.4",
"subtle",
]
[[package]]
name = "paste"
version = "1.0.15"
@@ -5400,6 +5469,17 @@ dependencies = [
"serde_core",
]
[[package]]
name = "sha1"
version = "0.10.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a978451301f4db1d02937a4ab3ccce137717b81826e79b7d49ffe3244a13c3b8"
dependencies = [
"cfg-if",
"cpufeatures",
"digest",
]
[[package]]
name = "sha2"
version = "0.10.9"
Cargo.toml
+3
View File
@@ -38,6 +38,9 @@ dac-engine-export = { path = "crates/engine-export" }
dac-ui-egui = { path = "crates/ui-egui" }
dac-mcp = { path = "crates/mcp" }
dac-fetch = { path = "crates/fetch" }
dac-net = { path = "crates/net" }
dac-credentials = { path = "crates/credentials" }
dac-hash = { path = "crates/hash" }
dac-heif = { path = "crates/heif" }
dac-segment = { path = "crates/segment" }
crates/credentials/Cargo.toml
+26
View File
@@ -0,0 +1,26 @@
[package]
name = "dac-credentials"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "Secret storage (L0): the OS keychain where reachable in pure Rust, else a passphrase-encrypted file"
[dependencies]
dac-brand = { workspace = true }
[lints]
workspace = true
# Native only: on wasm32 the crate is empty (the web build stores no credentials).
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
log = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
zeroize = "1"
getrandom = "0.3"
argon2 = { version = "0.5", default-features = false, features = ["alloc"] }
chacha20poly1305 = { version = "0.10", default-features = false, features = ["alloc"] }
# Linux and the BSDs: the freedesktop Secret Service over D-Bus, in pure Rust.
[target.'cfg(all(unix, not(target_os = "macos"), not(target_os = "ios"), not(target_os = "android")))'.dependencies]
zbus = { version = "5", default-features = false, features = ["blocking-api", "async-io"] }
crates/credentials/src/file.rs
+231
View File
@@ -0,0 +1,231 @@
//! The fallback store: one JSON file whose payload is the secrets map encrypted with
//! XChaCha20-Poly1305 under a key derived from a passphrase with Argon2id (RFC 9106).
//!
//! Sources: RFC 9106 (Argon2), RFC 8439 and draft-irtf-cfrg-xchacha (XChaCha20-Poly1305);
//! file layout is own design.
use std::collections::BTreeMap;
use std::io::Write;
use std::path::{Path, PathBuf};
use std::sync::{Mutex, PoisonError};
use chacha20poly1305::aead::{Aead, KeyInit, Payload};
use chacha20poly1305::{XChaCha20Poly1305, XNonce};
use serde::{Deserialize, Serialize};
use zeroize::Zeroizing;
use crate::{CredError, Key, Secret, SecretStore};
const FORMAT: u32 = 1;
/// Bound to the ciphertext so a payload can't be moved into another kind of file.
const AAD: &[u8] = b"credentials-file-v1";
/// Largest file read.
const MAX_FILE: u64 = 16 * 1024 * 1024;
/// Argon2id cost for new files (OWASP's minimum recommendation: 19 MiB, 2 passes).
const M_KIB: u32 = 19 * 1024;
const T: u32 = 2;
const P: u32 = 1;
#[derive(Serialize, Deserialize)]
struct Envelope {
format: u32,
kdf: String,
m_kib: u32,
t: u32,
p: u32,
salt: String,
nonce: String,
data: String,
}
fn hex(b: &[u8]) -> String {
b.iter().map(|b| format!("{b:02x}")).collect()
}
fn unhex(s: &str) -> Option<Vec<u8>> {
if !s.len().is_multiple_of(2) {
return None;
}
(0..s.len() / 2).map(|i| u8::from_str_radix(s.get(i * 2..i * 2 + 2)?, 16).ok()).collect()
}
fn random<const N: usize>() -> Result<[u8; N], CredError> {
let mut b = [0u8; N];
getrandom::fill(&mut b).map_err(|e| CredError::Io(format!("no randomness: {e}")))?;
Ok(b)
}
fn derive(passphrase: &Secret, salt: &[u8], m_kib: u32, t: u32, p: u32) -> Result<Zeroizing<[u8; 32]>, CredError> {
// bound the cost read from the file: a doctored header must not exhaust memory or time
if !(8..=1024 * 1024).contains(&m_kib) || !(1..=16).contains(&t) || !(1..=16).contains(&p) {
return Err(CredError::Corrupt("key derivation parameters out of range".into()));
}
let params = argon2::Params::new(m_kib, t, p, Some(32)).map_err(|e| CredError::Corrupt(e.to_string()))?;
let argon = argon2::Argon2::new(argon2::Algorithm::Argon2id, argon2::Version::V0x13, params);
let mut key = Zeroizing::new([0u8; 32]);
argon.hash_password_into(passphrase.expose().as_bytes(), salt, key.as_mut()).map_err(|e| CredError::Corrupt(e.to_string()))?;
Ok(key)
}
type Map = BTreeMap<String, String>;
fn map_key(k: &Key) -> String {
format!("{}\n{}", k.service, k.account)
}
/// Secrets in a passphrase-encrypted file. Every call reads and (for changes) rewrites the file,
/// so several handles on one file stay consistent.
pub struct FileStore {
path: PathBuf,
key: Zeroizing<[u8; 32]>,
salt: Vec<u8>,
lock: Mutex<()>,
}
impl std::fmt::Debug for FileStore {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("FileStore").field("path", &self.path).finish_non_exhaustive()
}
}
impl FileStore {
/// Open `path` with `passphrase`, or create an empty store there (written on the first
/// [`SecretStore::set`]). A wrong passphrase for an existing file is [`CredError::WrongPassphrase`].
pub fn open(path: &Path, passphrase: &Secret) -> Result<FileStore, CredError> {
match read_envelope(path)? {
Some(env) => {
let salt = unhex(&env.salt).ok_or_else(|| CredError::Corrupt("bad salt".into()))?;
let key = derive(passphrase, &salt, env.m_kib, env.t, env.p)?;
let store = FileStore { path: path.to_path_buf(), key, salt, lock: Mutex::new(()) };
store.decrypt(&env)?; // proves the passphrase
Ok(store)
}
None => {
let salt = random::<16>()?.to_vec();
let key = derive(passphrase, &salt, M_KIB, T, P)?;
Ok(FileStore { path: path.to_path_buf(), key, salt, lock: Mutex::new(()) })
}
}
}
pub fn path(&self) -> &Path {
&self.path
}
fn cipher(&self) -> Result<XChaCha20Poly1305, CredError> {
XChaCha20Poly1305::new_from_slice(self.key.as_ref()).map_err(|_| CredError::Corrupt("bad key length".into()))
}
fn decrypt(&self, env: &Envelope) -> Result<Map, CredError> {
if env.format != FORMAT || env.kdf != "argon2id" {
return Err(CredError::Corrupt(format!("unknown format {}", env.format)));
}
let nonce: [u8; 24] = unhex(&env.nonce).and_then(|n| n.try_into().ok()).ok_or_else(|| CredError::Corrupt("bad nonce".into()))?;
let data = unhex(&env.data).ok_or_else(|| CredError::Corrupt("bad payload".into()))?;
let plain =
Zeroizing::new(self.cipher()?.decrypt(&XNonce::from(nonce), Payload { msg: &data, aad: AAD }).map_err(|_| CredError::WrongPassphrase)?);
let map: Map = serde_json::from_slice(&plain).map_err(|_| CredError::Corrupt("bad payload".into()))?;
Ok(map)
}
fn load(&self) -> Result<Map, CredError> {
match read_envelope(&self.path)? {
Some(env) => {
if unhex(&env.salt).as_deref() != Some(&self.salt[..]) {
return Err(CredError::Corrupt("the file was replaced while open; open it again".into()));
}
self.decrypt(&env)
}
None => Ok(Map::new()),
}
}
fn save(&self, map: &Map) -> Result<(), CredError> {
let plain = Zeroizing::new(serde_json::to_vec(map).map_err(|e| CredError::Io(e.to_string()))?);
let nonce = random::<24>()?;
let data =
self.cipher()?.encrypt(&XNonce::from(nonce), Payload { msg: &plain, aad: AAD }).map_err(|_| CredError::Io("encryption failed".into()))?;
let env = Envelope {
format: FORMAT,
kdf: "argon2id".into(),
m_kib: M_KIB,
t: T,
p: P,
salt: hex(&self.salt),
nonce: hex(&nonce),
data: hex(&data),
};
let text = serde_json::to_vec_pretty(&env).map_err(|e| CredError::Io(e.to_string()))?;
write_atomic(&self.path, &text)
}
}
fn io(path: &Path, e: std::io::Error) -> CredError {
CredError::Io(format!("{}: {e}", path.display()))
}
fn read_envelope(path: &Path) -> Result<Option<Envelope>, CredError> {
let meta = match std::fs::metadata(path) {
Ok(m) => m,
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None),
Err(e) => return Err(io(path, e)),
};
if meta.len() > MAX_FILE {
return Err(CredError::Corrupt("file too large".into()));
}
let bytes = std::fs::read(path).map_err(|e| io(path, e))?;
serde_json::from_slice(&bytes).map(Some).map_err(|_| CredError::Corrupt("not a credentials file".into()))
}
/// Write to a temporary file (owner-only on Unix), sync, then rename over `path`.
fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), CredError> {
if let Some(dir) = path.parent().filter(|p| !p.as_os_str().is_empty()) {
std::fs::create_dir_all(dir).map_err(|e| io(dir, e))?;
}
let mut name = path.file_name().map(|n| n.to_os_string()).unwrap_or_default();
name.push(".tmp");
let tmp = path.with_file_name(name);
let mut opts = std::fs::OpenOptions::new();
opts.write(true).create(true).truncate(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
opts.mode(0o600);
}
let mut f = opts.open(&tmp).map_err(|e| io(&tmp, e))?;
f.write_all(bytes).and_then(|_| f.sync_all()).map_err(|e| io(&tmp, e))?;
drop(f);
std::fs::rename(&tmp, path).map_err(|e| io(path, e))
}
impl SecretStore for FileStore {
fn name(&self) -> &'static str {
"encrypted file"
}
fn get(&self, key: &Key) -> Result<Option<Secret>, CredError> {
let _g = self.lock.lock().unwrap_or_else(PoisonError::into_inner);
Ok(self.load()?.get(&map_key(key)).map(|s| Secret::new(s.clone())))
}
fn set(&self, key: &Key, secret: &Secret) -> Result<(), CredError> {
let _g = self.lock.lock().unwrap_or_else(PoisonError::into_inner);
let mut map = self.load()?;
map.insert(map_key(key), secret.expose().to_string());
self.save(&map)?;
log::info!("credentials: stored a secret for {} in the encrypted file", key.service);
Ok(())
}
fn delete(&self, key: &Key) -> Result<bool, CredError> {
let _g = self.lock.lock().unwrap_or_else(PoisonError::into_inner);
let mut map = self.load()?;
let removed = map.remove(&map_key(key)).is_some();
if removed {
self.save(&map)?;
log::info!("credentials: removed a secret for {} from the encrypted file", key.service);
}
Ok(removed)
}
}
crates/credentials/src/lib.rs
+118
View File
@@ -0,0 +1,118 @@
//! Where account secrets (API keys, tokens, passwords) live: never in the catalog, settings,
//! logs, MCP output or crash reports.
//!
//! - [`os_store`]: the platform keychain, where it is reachable in pure Rust without `unsafe`.
//! Linux and the BSDs use the freedesktop Secret Service (GNOME Keyring, KWallet) over D-Bus
//! with `zbus`. macOS Keychain and Windows Credential Manager need FFI, which this workspace
//! keeps out of product crates (see AGENTS.md: `unsafe` lives only in `crates/sysmem`), so there
//! [`os_store`] answers [`CredError::Unsupported`] and callers use the file store. The platform
//! code is isolated in `platform/`, so a safe backend can drop in later.
//! - [`FileStore`]: a file encrypted with XChaCha20-Poly1305 under a key derived from the user's
//! passphrase with Argon2id. Wrong passphrases and tampering are detected, never "decrypted"
//! to garbage; writes are atomic.
//!
//! [`Secret`] holds a secret in memory: its `Debug`/`Display` never show the value and the memory
//! is zeroed on drop. Read it with [`Secret::expose`] only where it is sent.
//!
//! Native only: on wasm32 this crate is empty.
#![cfg(not(target_arch = "wasm32"))]
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
mod file;
mod platform;
pub use file::FileStore;
use zeroize::Zeroizing;
/// A secret value. Never printed: `{:?}` and `{}` show `<redacted>`.
#[derive(Clone, PartialEq, Eq)]
pub struct Secret(Zeroizing<String>);
impl Secret {
pub fn new(value: impl Into<String>) -> Secret {
Secret(Zeroizing::new(value.into()))
}
/// The value, for the one place that sends it.
pub fn expose(&self) -> &str {
&self.0
}
}
impl std::fmt::Debug for Secret {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("Secret(<redacted>)")
}
}
impl std::fmt::Display for Secret {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str("<redacted>")
}
}
/// Which secret: a service (`"immich"`) and an account within it (e.g. `server URL + user`).
#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct Key {
pub service: String,
pub account: String,
}
impl Key {
pub fn new(service: &str, account: &str) -> Key {
Key { service: service.to_string(), account: account.to_string() }
}
}
/// Why a secret could not be read or written. Messages never contain secrets.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum CredError {
/// No keychain backend on this platform: use [`FileStore`].
Unsupported(&'static str),
/// The keychain exists but can't be reached (no session bus, no Secret Service running…).
Unavailable(String),
/// The keychain is locked and unlocking needs a prompt.
Locked,
/// The passphrase does not open the file (or the file was altered).
WrongPassphrase,
/// The file is not a credentials file of a known format.
Corrupt(String),
Io(String),
}
impl std::fmt::Display for CredError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
CredError::Unsupported(e) => write!(f, "no system keychain available: {e}"),
CredError::Unavailable(e) => write!(f, "the system keychain can't be reached: {e}"),
CredError::Locked => write!(f, "the system keychain is locked; unlock it and try again"),
CredError::WrongPassphrase => write!(f, "wrong passphrase, or the credentials file was altered"),
CredError::Corrupt(e) => write!(f, "unreadable credentials file: {e}"),
CredError::Io(e) => write!(f, "{e}"),
}
}
}
impl std::error::Error for CredError {}
/// A place to keep secrets.
pub trait SecretStore: Send + Sync {
/// Short description for settings ("Secret Service", "encrypted file").
fn name(&self) -> &'static str;
fn get(&self, key: &Key) -> Result<Option<Secret>, CredError>;
/// Store or replace.
fn set(&self, key: &Key, secret: &Secret) -> Result<(), CredError>;
/// `true` when something was deleted.
fn delete(&self, key: &Key) -> Result<bool, CredError>;
}
/// The platform keychain, or why there is none.
pub fn os_store() -> Result<Box<dyn SecretStore>, CredError> {
platform::open()
}
#[cfg(test)]
mod tests;
crates/credentials/src/platform/mod.rs
+18
View File
@@ -0,0 +1,18 @@
//! Platform keychains, one module per backend. Everything platform-specific stays in here.
#[cfg(all(unix, not(target_os = "macos"), not(target_os = "ios"), not(target_os = "android")))]
mod secret_service;
use crate::{CredError, SecretStore};
#[cfg(all(unix, not(target_os = "macos"), not(target_os = "ios"), not(target_os = "android")))]
pub(crate) fn open() -> Result<Box<dyn SecretStore>, CredError> {
Ok(Box::new(secret_service::SecretService::connect()?))
}
/// macOS Keychain and Windows Credential Manager are only reachable through FFI, which product
/// crates may not use (AGENTS.md). Until a safe binding exists, callers use the file store.
#[cfg(not(all(unix, not(target_os = "macos"), not(target_os = "ios"), not(target_os = "android"))))]
pub(crate) fn open() -> Result<Box<dyn SecretStore>, CredError> {
Err(CredError::Unsupported("this platform's keychain needs native code; use an encrypted file"))
}
crates/credentials/src/platform/secret_service.rs
+147
View File
@@ -0,0 +1,147 @@
//! The freedesktop Secret Service API (GNOME Keyring, KWallet, KeePassXC) over the D-Bus session
//! bus, with `zbus`'s blocking API.
//!
//! Sources: the freedesktop.org Secret Service API specification (0.2). Own implementation.
//!
//! Items carry the attributes `application` (the app id from `dac_brand`), `service` and
//! `account`; lookups match all three. The session uses the `plain` algorithm: the secret travels
//! over the local session bus to the keyring daemon, as with most clients. A locked collection
//! that needs an unlock prompt is reported as [`CredError::Locked`].
use std::collections::HashMap;
use zbus::blocking::{Connection, Proxy};
use zbus::zvariant::{ObjectPath, OwnedObjectPath, OwnedValue, Value};
use crate::{CredError, Key, Secret, SecretStore};
const DEST: &str = "org.freedesktop.secrets";
const SERVICE_PATH: &str = "/org/freedesktop/secrets";
const SERVICE: &str = "org.freedesktop.Secret.Service";
const COLLECTION: &str = "org.freedesktop.Secret.Collection";
const ITEM: &str = "org.freedesktop.Secret.Item";
const SESSION: &str = "org.freedesktop.Secret.Session";
/// The Secret struct `(oayays)`: session, parameters, value, content type.
type WireSecret = (OwnedObjectPath, Vec<u8>, Vec<u8>, String);
fn unavailable(e: zbus::Error) -> CredError {
CredError::Unavailable(e.to_string())
}
pub(crate) struct SecretService {
conn: Connection,
}
/// An open `plain` session, closed on drop.
struct Session<'a> {
conn: &'a Connection,
path: OwnedObjectPath,
}
impl Drop for Session<'_> {
fn drop(&mut self) {
if let Ok(p) = Proxy::new(self.conn, DEST, self.path.as_ref(), SESSION) {
let _ = p.call_method("Close", &());
}
}
}
fn is_root(p: &ObjectPath<'_>) -> bool {
p.as_str() == "/"
}
impl SecretService {
pub(crate) fn connect() -> Result<SecretService, CredError> {
let conn = Connection::session().map_err(unavailable)?;
let svc = SecretService { conn };
// fail now, not at first use, when nothing provides the service
svc.service()?.call::<_, _, OwnedObjectPath>("ReadAlias", &"default").map_err(unavailable)?;
Ok(svc)
}
fn service(&self) -> Result<Proxy<'_>, CredError> {
Proxy::new(&self.conn, DEST, SERVICE_PATH, SERVICE).map_err(unavailable)
}
fn proxy<'a>(&'a self, path: &'a OwnedObjectPath, iface: &'static str) -> Result<Proxy<'a>, CredError> {
Proxy::new(&self.conn, DEST, path.as_ref(), iface).map_err(unavailable)
}
fn session(&self) -> Result<Session<'_>, CredError> {
let (_, path): (OwnedValue, OwnedObjectPath) = self.service()?.call("OpenSession", &("plain", Value::from(""))).map_err(unavailable)?;
Ok(Session { conn: &self.conn, path })
}
fn attributes(key: &Key) -> HashMap<String, String> {
HashMap::from([
("application".to_string(), dac_brand::APP_ID.to_string()),
("service".to_string(), key.service.clone()),
("account".to_string(), key.account.clone()),
])
}
/// Matching items, unlocked.
fn find(&self, key: &Key) -> Result<Vec<OwnedObjectPath>, CredError> {
let (unlocked, locked): (Vec<OwnedObjectPath>, Vec<OwnedObjectPath>) =
self.service()?.call("SearchItems", &Self::attributes(key)).map_err(unavailable)?;
if locked.is_empty() {
return Ok(unlocked);
}
let (mut now, prompt): (Vec<OwnedObjectPath>, OwnedObjectPath) = self.service()?.call("Unlock", &locked).map_err(unavailable)?;
if !is_root(&prompt) && now.is_empty() {
return Err(CredError::Locked);
}
now.extend(unlocked);
Ok(now)
}
}
impl SecretStore for SecretService {
fn name(&self) -> &'static str {
"Secret Service"
}
fn get(&self, key: &Key) -> Result<Option<Secret>, CredError> {
let Some(item) = self.find(key)?.into_iter().next() else { return Ok(None) };
let session = self.session()?;
let (_, _, value, _): WireSecret = self.proxy(&item, ITEM)?.call("GetSecret", &session.path).map_err(unavailable)?;
let value = zeroize::Zeroizing::new(value);
let text = std::str::from_utf8(&value).map_err(|_| CredError::Corrupt("the stored secret is not text".into()))?;
Ok(Some(Secret::new(text)))
}
fn set(&self, key: &Key, secret: &Secret) -> Result<(), CredError> {
let collection: OwnedObjectPath = self.service()?.call("ReadAlias", &"default").map_err(unavailable)?;
if is_root(&collection) {
return Err(CredError::Unavailable("the keyring has no default collection".into()));
}
let session = self.session()?;
let label = format!("{} – {} ({})", dac_brand::DISPLAY_NAME, key.service, key.account);
let mut props: HashMap<&str, Value<'_>> = HashMap::new();
props.insert("org.freedesktop.Secret.Item.Label", Value::from(label));
props.insert("org.freedesktop.Secret.Item.Attributes", Value::from(Self::attributes(key)));
let wire: WireSecret = (session.path.clone(), Vec::new(), secret.expose().as_bytes().to_vec(), "text/plain; charset=utf8".into());
let (item, prompt): (OwnedObjectPath, OwnedObjectPath) =
self.proxy(&collection, COLLECTION)?.call("CreateItem", &(props, wire, true)).map_err(unavailable)?;
if is_root(&item) && !is_root(&prompt) {
return Err(CredError::Locked);
}
log::info!("credentials: stored a secret for {} in the Secret Service", key.service);
Ok(())
}
fn delete(&self, key: &Key) -> Result<bool, CredError> {
let items = self.find(key)?;
for item in &items {
let prompt: OwnedObjectPath = self.proxy(item, ITEM)?.call("Delete", &()).map_err(unavailable)?;
if !is_root(&prompt) {
return Err(CredError::Locked);
}
}
if !items.is_empty() {
log::info!("credentials: removed a secret for {} from the Secret Service", key.service);
}
Ok(!items.is_empty())
}
}
crates/credentials/src/tests.rs
+135
View File
@@ -0,0 +1,135 @@
use std::sync::Mutex;
use super::*;
const SECRET: &str = "immich-api-key-7f3a9c";
fn tempdir(tag: &str) -> std::path::PathBuf {
let d = std::env::temp_dir().join(format!("dac-cred-test-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
struct Capture;
static LOGS: Mutex<String> = Mutex::new(String::new());
impl log::Log for Capture {
fn enabled(&self, _: &log::Metadata) -> bool {
true
}
fn log(&self, r: &log::Record) {
LOGS.lock().unwrap().push_str(&format!("{} {}\n", r.target(), r.args()));
}
fn flush(&self) {}
}
fn capture_logs() {
static ONCE: std::sync::Once = std::sync::Once::new();
ONCE.call_once(|| {
let _ = log::set_logger(&Capture);
log::set_max_level(log::LevelFilter::Trace);
});
}
#[test]
fn file_store_round_trip_and_wrong_passphrase() {
let dir = tempdir("rt");
let path = dir.join("sub/credentials.json");
let pass = Secret::new("correct horse");
let key = Key::new("immich", "https://photos.home|me@example.org");
{
let s = FileStore::open(&path, &pass).unwrap();
assert_eq!(s.get(&key).unwrap(), None);
s.set(&key, &Secret::new(SECRET)).unwrap();
s.set(&Key::new("other", "x"), &Secret::new("y")).unwrap();
}
let s = FileStore::open(&path, &pass).unwrap();
assert_eq!(s.get(&key).unwrap().unwrap().expose(), SECRET);
assert!(s.delete(&Key::new("other", "x")).unwrap());
assert!(!s.delete(&Key::new("other", "x")).unwrap());
assert_eq!(s.get(&key).unwrap().unwrap().expose(), SECRET);
// the file never holds the plaintext
let raw = std::fs::read_to_string(&path).unwrap();
assert!(!raw.contains(SECRET) && !raw.contains("photos.home"));
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
assert_eq!(std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o600);
}
assert_eq!(FileStore::open(&path, &Secret::new("wrong")).unwrap_err(), CredError::WrongPassphrase);
let _ = std::fs::remove_dir_all(dir);
}
#[test]
fn hostile_files_are_errors() {
let dir = tempdir("hostile");
let pass = Secret::new("p");
let path = dir.join("c.json");
let s = FileStore::open(&path, &pass).unwrap();
s.set(&Key::new("a", "b"), &Secret::new("v")).unwrap();
let good = std::fs::read_to_string(&path).unwrap();
// flipped ciphertext byte
let mut env: serde_json::Value = serde_json::from_str(&good).unwrap();
let data = env["data"].as_str().unwrap().to_string();
let flipped = format!("{}{}", if data.starts_with('0') { "1" } else { "0" }, &data[1..]);
env["data"] = flipped.into();
std::fs::write(&path, env.to_string()).unwrap();
assert_eq!(FileStore::open(&path, &pass).unwrap_err(), CredError::WrongPassphrase);
// absurd KDF cost
let mut env: serde_json::Value = serde_json::from_str(&good).unwrap();
env["m_kib"] = (u32::MAX).into();
std::fs::write(&path, env.to_string()).unwrap();
assert!(matches!(FileStore::open(&path, &pass), Err(CredError::Corrupt(_))));
for junk in ["", "{}", "not json", r#"{"format":1,"kdf":"argon2id","m_kib":64,"t":1,"p":1,"salt":"zz","nonce":"","data":""}"#] {
std::fs::write(&path, junk).unwrap();
assert!(FileStore::open(&path, &pass).is_err(), "{junk}");
}
let _ = std::fs::remove_dir_all(dir);
}
#[test]
fn secrets_never_reach_debug_output_or_logs() {
capture_logs();
let secret = Secret::new(SECRET);
assert_eq!(format!("{secret:?}"), "Secret(<redacted>)");
assert_eq!(format!("{secret}"), "<redacted>");
let dir = tempdir("logs");
let path = dir.join("c.json");
let s = FileStore::open(&path, &Secret::new("passphrase-xyz")).unwrap();
let key = Key::new("immich", "acct");
s.set(&key, &secret).unwrap();
let got = s.get(&key).unwrap();
s.delete(&key).unwrap();
let wrong = FileStore::open(&path, &Secret::new("nope")).err();
let debug = format!("{s:?} {got:?} {wrong:?}");
// the OS store may or may not exist here; either way its output must not leak
let os = os_store().map(|st| st.name()).map_err(|e| e.to_string());
let logs = LOGS.lock().unwrap().clone();
assert!(logs.contains("credentials: stored a secret"), "logging works: {logs}");
for text in [&logs, &debug, &format!("{os:?}")] {
assert!(!text.contains(SECRET), "secret leaked into {text}");
assert!(!text.contains("passphrase-xyz"), "passphrase leaked into {text}");
}
let _ = std::fs::remove_dir_all(dir);
}
/// Exercises the real keychain only when asked (it may prompt or touch the user's keyring).
#[test]
fn os_store_round_trip_when_enabled() {
if std::env::var_os("DAC_TEST_OS_KEYCHAIN").is_none() {
return;
}
let store = os_store().unwrap();
let key = Key::new("test-service", &format!("test-{}", std::process::id()));
store.set(&key, &Secret::new(SECRET)).unwrap();
assert_eq!(store.get(&key).unwrap().unwrap().expose(), SECRET);
assert!(store.delete(&key).unwrap());
assert_eq!(store.get(&key).unwrap(), None);
}
crates/hash/Cargo.toml
+12
View File
@@ -0,0 +1,12 @@
[package]
name = "dac-hash"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "Streaming content hashes (L0): SHA-1 of originals, as remote services such as Immich store them"
[dependencies]
sha1 = { version = "0.10", default-features = false }
[lints]
workspace = true
crates/hash/src/lib.rs
+167
View File
@@ -0,0 +1,167 @@
//! Streaming SHA-1 of a file's bytes, for matching originals with remote services that identify
//! assets by it (Immich stores the SHA-1 of every original). SHA-1 is used here as a content
//! identifier agreed with those services, not for security.
//!
//! Made to ride along an existing read pass: wrap the reader the import already uses in a
//! [`HashingReader`] (or feed the chunks it reads to a [`Sha1`]) and take the digest at the end,
//! so a file is read from disk once. [`sha1_reader`] and [`sha1_file`] cover a standalone pass
//! (back-filling old catalogs).
//!
//! Pure computation; builds everywhere, wasm32 included.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
use std::io::Read;
use std::path::Path;
use sha1::Digest;
/// A SHA-1 digest.
#[derive(Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct Sha1Digest(pub [u8; 20]);
impl Sha1Digest {
/// Lower-case hex (40 characters).
pub fn to_hex(&self) -> String {
self.0.iter().map(|b| format!("{b:02x}")).collect()
}
/// Parse 40 hex characters (any case).
pub fn from_hex(s: &str) -> Option<Sha1Digest> {
let s = s.trim();
if s.len() != 40 {
return None;
}
let mut out = [0u8; 20];
for (i, b) in out.iter_mut().enumerate() {
*b = u8::from_str_radix(s.get(i * 2..i * 2 + 2)?, 16).ok()?;
}
Some(Sha1Digest(out))
}
/// Standard base64, the form Immich's API uses for `checksum`.
pub fn to_base64(&self) -> String {
const A: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
let mut out = String::with_capacity(28);
for chunk in self.0.chunks(3) {
let b = [chunk.first().copied().unwrap_or(0), chunk.get(1).copied().unwrap_or(0), chunk.get(2).copied().unwrap_or(0)];
let n = (u32::from(b[0]) << 16) | (u32::from(b[1]) << 8) | u32::from(b[2]);
for i in 0..4 {
if i <= chunk.len() {
out.push(char::from(A.get(((n >> (18 - 6 * i)) & 63) as usize).copied().unwrap_or(b'=')));
} else {
out.push('=');
}
}
}
out
}
}
impl std::fmt::Display for Sha1Digest {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.to_hex())
}
}
impl std::fmt::Debug for Sha1Digest {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Sha1Digest({})", self.to_hex())
}
}
/// An incremental SHA-1.
#[derive(Clone, Default)]
pub struct Sha1(sha1::Sha1);
impl Sha1 {
pub fn new() -> Sha1 {
Sha1::default()
}
pub fn update(&mut self, bytes: &[u8]) {
self.0.update(bytes);
}
pub fn finish(self) -> Sha1Digest {
Sha1Digest(self.0.finalize().into())
}
}
/// Hashes everything read through it.
pub struct HashingReader<R> {
inner: R,
hasher: Sha1,
bytes: u64,
}
impl<R: Read> HashingReader<R> {
pub fn new(inner: R) -> Self {
HashingReader { inner, hasher: Sha1::new(), bytes: 0 }
}
/// Bytes read so far.
pub fn bytes_read(&self) -> u64 {
self.bytes
}
/// The digest of everything read so far, and the reader back. Read to the end first for the
/// whole file's SHA-1.
pub fn finish(self) -> (R, Sha1Digest) {
(self.inner, self.hasher.finish())
}
}
impl<R: Read> Read for HashingReader<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.inner.read(buf)?;
self.hasher.update(buf.get(..n).unwrap_or_default());
self.bytes = self.bytes.saturating_add(n as u64);
Ok(n)
}
}
/// SHA-1 of everything `r` yields.
pub fn sha1_reader(r: impl Read) -> std::io::Result<Sha1Digest> {
let mut h = HashingReader::new(r);
std::io::copy(&mut h, &mut std::io::sink())?;
Ok(h.finish().1)
}
/// SHA-1 of a file's bytes.
pub fn sha1_file(path: &Path) -> std::io::Result<Sha1Digest> {
sha1_reader(std::io::BufReader::with_capacity(1 << 20, std::fs::File::open(path)?))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn known_vectors() {
assert_eq!(sha1_reader(&b""[..]).unwrap().to_hex(), "da39a3ee5e6b4b0d3255bfef95601890afd80709");
let abc = sha1_reader(&b"abc"[..]).unwrap();
assert_eq!(abc.to_hex(), "a9993e364706816aba3e25717850c26c9cd0d89d");
assert_eq!(abc.to_base64(), "qZk+NkcGgWq6PiVxeFDCbJzQ2J0=");
assert_eq!(Sha1Digest::from_hex(&abc.to_hex().to_uppercase()), Some(abc));
assert_eq!(Sha1Digest::from_hex("xyz"), None);
assert_eq!(Sha1Digest::from_hex(&"é".repeat(20)), None);
}
#[test]
fn hashing_reader_rides_along() {
let data: Vec<u8> = (0..100_000u32).map(|i| i as u8).collect();
let mut r = HashingReader::new(&data[..]);
let mut copy = Vec::new();
r.read_to_end(&mut copy).unwrap();
assert_eq!(r.bytes_read(), 100_000);
let (_, d) = r.finish();
let mut h = Sha1::new();
for c in data.chunks(777) {
h.update(c);
}
assert_eq!(h.finish(), d);
assert_eq!(copy, data);
}
}
crates/net/Cargo.toml
+23
View File
@@ -0,0 +1,23 @@
[package]
name = "dac-net"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "HTTP(S) client in pure Rust (L0): JSON, multipart upload, resumable verified downloads, proxy, per-connection trust"
[dependencies]
dac-brand = { workspace = true }
[lints]
workspace = true
# Native only: on wasm32 the crate is empty (the web build makes no requests through it).
[target.'cfg(not(target_arch = "wasm32"))'.dependencies]
log = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
rustls = { version = "0.23", default-features = false, features = ["std", "tls12"] }
rustls-rustcrypto = "0.0.2-alpha"
webpki-roots = "1"
sha2 = { version = "0.10", default-features = false, features = ["std"] }
flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
crates/net/src/client.rs
+414
View File
@@ -0,0 +1,414 @@
//! The client: configuration, the request builder and the response.
use std::io::{BufReader, BufWriter, Read, Write};
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::time::Duration;
use serde::Serialize;
use serde::de::DeserializeOwned;
use crate::conn::{self, ConnectOptions};
use crate::http::{self, BodyReader, Framing};
use crate::{Multipart, NetError, Trust, Url};
/// HTTP method.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Method {
Get,
Head,
Post,
Put,
Patch,
Delete,
}
impl Method {
pub fn as_str(self) -> &'static str {
match self {
Method::Get => "GET",
Method::Head => "HEAD",
Method::Post => "POST",
Method::Put => "PUT",
Method::Patch => "PATCH",
Method::Delete => "DELETE",
}
}
}
/// Header names that carry credentials: never sent to another origin after a redirect.
const SENSITIVE: &[&str] = &["authorization", "proxy-authorization", "cookie", "x-api-key"];
/// Settings shared by every request of a [`Client`].
#[derive(Debug, Clone)]
pub struct ClientConfig {
/// Per address tried.
pub connect_timeout: Duration,
/// Longest wait for any progress while sending or receiving.
pub stall_timeout: Duration,
/// Redirects followed per request (0: none, the 3xx response is returned).
pub max_redirects: u8,
/// `http://host:port` of an HTTP proxy (HTTPS goes through it with `CONNECT`).
pub proxy: Option<String>,
/// Which server certificates are trusted.
pub trust: Trust,
/// Cap for [`Response::bytes`], [`Response::text`] and [`Response::json`].
pub max_body: u64,
}
impl Default for ClientConfig {
fn default() -> Self {
ClientConfig {
connect_timeout: Duration::from_secs(10),
stall_timeout: Duration::from_secs(30),
max_redirects: 5,
proxy: None,
trust: Trust::new(),
max_body: 64 * 1024 * 1024,
}
}
}
/// An HTTP(S) client. Blocking: run it off the UI thread. Cheap to clone.
#[derive(Clone)]
pub struct Client {
config: ClientConfig,
proxy: Option<Url>,
tls: Arc<rustls::ClientConfig>,
}
impl std::fmt::Debug for Client {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Client").field("config", &self.config).finish()
}
}
impl Client {
pub fn new(config: ClientConfig) -> Result<Client, NetError> {
let proxy = match &config.proxy {
Some(p) => {
let u = Url::parse(p)?;
if u.tls {
return Err(NetError::BadUrl("only http:// proxies are supported".into()));
}
Some(u)
}
None => None,
};
let tls = config.trust.client_config()?;
Ok(Client { config, proxy, tls })
}
pub fn config(&self) -> &ClientConfig {
&self.config
}
pub fn request(&self, method: Method, url: &str) -> Request<'_> {
Request {
client: self,
method,
url: Url::parse(url),
headers: Vec::new(),
secret_headers: Vec::new(),
body: Body::Empty,
cancel: None,
progress: None,
error: None,
accept_gzip: true,
}
}
pub fn get(&self, url: &str) -> Request<'_> {
self.request(Method::Get, url)
}
pub fn head(&self, url: &str) -> Request<'_> {
self.request(Method::Head, url)
}
pub fn post(&self, url: &str) -> Request<'_> {
self.request(Method::Post, url)
}
pub fn put(&self, url: &str) -> Request<'_> {
self.request(Method::Put, url)
}
pub fn patch(&self, url: &str) -> Request<'_> {
self.request(Method::Patch, url)
}
pub fn delete(&self, url: &str) -> Request<'_> {
self.request(Method::Delete, url)
}
}
enum Body {
Empty,
Bytes { data: Vec<u8>, content_type: String },
Multipart(Multipart),
}
/// Progress callback: `(bytes sent, total)` while uploading.
pub type Progress<'a> = &'a mut dyn FnMut(u64, Option<u64>);
/// A request under construction; [`Request::send`] performs it.
pub struct Request<'a> {
client: &'a Client,
method: Method,
url: Result<Url, NetError>,
headers: Vec<(String, String)>,
secret_headers: Vec<String>,
body: Body,
cancel: Option<Arc<AtomicBool>>,
progress: Option<Progress<'a>>,
error: Option<NetError>,
accept_gzip: bool,
}
fn check_header(name: &str, value: &str) -> Result<(), NetError> {
let bad_name = name.is_empty() || name.chars().any(|c| !c.is_ascii_graphic() || c == ':');
if bad_name || value.chars().any(|c| c == '\r' || c == '\n' || c == '\0') {
// the value is never put in the message: it may be a secret
return Err(NetError::BadHeader(format!("{:?}", name.chars().take(64).collect::<String>())));
}
Ok(())
}
impl<'a> Request<'a> {
/// Add a header.
pub fn header(mut self, name: &str, value: &str) -> Self {
match check_header(name, value) {
Ok(()) => self.headers.push((name.to_string(), value.to_string())),
Err(e) => self.error = self.error.or(Some(e)),
}
self
}
/// Add a header holding a credential: it is dropped when a redirect leaves the origin.
pub fn secret_header(mut self, name: &str, value: &str) -> Self {
self.secret_headers.push(name.to_ascii_lowercase());
self.header(name, value)
}
/// `Authorization: Bearer …`.
pub fn bearer(self, token: &str) -> Self {
self.secret_header("Authorization", &format!("Bearer {token}"))
}
/// A raw body.
pub fn body(mut self, content_type: &str, data: Vec<u8>) -> Self {
self.body = Body::Bytes { data, content_type: content_type.to_string() };
self
}
/// A JSON body.
pub fn json<T: Serialize + ?Sized>(mut self, value: &T) -> Self {
match serde_json::to_vec(value) {
Ok(data) => self.body = Body::Bytes { data, content_type: "application/json".into() },
Err(e) => self.error = self.error.or(Some(NetError::Json(e.to_string()))),
}
self
}
/// A streaming `multipart/form-data` body.
pub fn multipart(mut self, form: Multipart) -> Self {
self.body = Body::Multipart(form);
self
}
/// Abort (with [`NetError::Cancelled`]) once this flag is raised, also while reading the
/// response body.
pub fn cancel(mut self, flag: Arc<AtomicBool>) -> Self {
self.cancel = Some(flag);
self
}
/// Report upload progress.
pub fn progress(mut self, f: Progress<'a>) -> Self {
self.progress = Some(f);
self
}
/// Don't ask for a compressed response (downloads with `Range`).
pub fn identity_encoding(mut self) -> Self {
self.accept_gzip = false;
self
}
/// Perform the request, following redirects. Any status is a successful `send`; see
/// [`Response::error_for_status`].
pub fn send(self) -> Result<Response, NetError> {
let Request { client, mut method, url, mut headers, secret_headers, mut body, cancel, mut progress, error, accept_gzip } = self;
if let Some(e) = error {
return Err(e);
}
let mut url = url?;
let origin = url.origin();
let mut redirects = 0u8;
loop {
let path_only = url.path.split('?').next().unwrap_or("/");
log::debug!("net: {} {}{}", method.as_str(), url.origin(), path_only);
let opts = ConnectOptions {
connect_timeout: client.config.connect_timeout,
stall: client.config.stall_timeout,
cancel: cancel.clone(),
proxy: client.proxy.as_ref(),
tls: Some(client.tls.clone()),
};
let conn = conn::connect(&url, opts)?;
let mut w = BufWriter::with_capacity(64 * 1024, conn);
let target = if client.proxy.is_some() && !url.tls { url.to_string() } else { url.path.clone() };
let mut head = format!(
"{} {target} HTTP/1.1\r\nHost: {}\r\nUser-Agent: {}\r\nConnection: close\r\n",
method.as_str(),
url.host_header(),
crate::user_agent()
);
let has = |n: &str| headers.iter().any(|(k, _)| k.eq_ignore_ascii_case(n));
if !has("accept") {
head.push_str("Accept: */*\r\n");
}
if !has("accept-encoding") {
head.push_str(if accept_gzip { "Accept-Encoding: gzip\r\n" } else { "Accept-Encoding: identity\r\n" });
}
let total = match &body {
Body::Empty => (method != Method::Get && method != Method::Head && method != Method::Delete).then_some(0),
Body::Bytes { data, .. } => Some(data.len() as u64),
Body::Multipart(m) => Some(m.len()),
};
if let Some(n) = total {
head.push_str(&format!("Content-Length: {n}\r\n"));
}
match &body {
Body::Bytes { content_type, .. } if !has("content-type") => head.push_str(&format!("Content-Type: {content_type}\r\n")),
Body::Multipart(m) => head.push_str(&format!("Content-Type: {}\r\n", m.content_type())),
_ => {}
}
for (k, v) in &headers {
head.push_str(&format!("{k}: {v}\r\n"));
}
head.push_str("\r\n");
w.write_all(head.as_bytes())?;
let replay = match std::mem::replace(&mut body, Body::Empty) {
Body::Empty => Some(Body::Empty),
Body::Bytes { data, content_type } => {
for chunk in data.chunks(64 * 1024) {
w.write_all(chunk)?;
}
if let Some(p) = progress.as_mut() {
p(data.len() as u64, total);
}
Some(Body::Bytes { data, content_type })
}
Body::Multipart(m) => {
let mut report = |sent: u64| {
if let Some(p) = progress.as_mut() {
p(sent, total);
}
};
m.write_to(&mut w, &mut report)?;
None
}
};
w.flush()?;
let conn = w.into_inner().map_err(|e| NetError::Io(e.to_string()))?;
let mut r = BufReader::with_capacity(64 * 1024, conn);
let head = http::read_head(&mut r)?;
let location = head.header("location").filter(|_| matches!(head.status, 301 | 302 | 303 | 307 | 308));
if let Some(location) = location {
if redirects >= client.config.max_redirects {
if client.config.max_redirects == 0 {
return Response::new(head, r, url, method, client.config.max_body);
}
return Err(NetError::Redirect(format!("more than {} redirects", client.config.max_redirects)));
}
redirects += 1;
let next = url.join(location)?;
if url.tls && !next.tls {
return Err(NetError::Redirect("refusing to follow a redirect from https to http".into()));
}
if head.status == 303 || ((head.status == 301 || head.status == 302) && method == Method::Post) {
method = if method == Method::Head { Method::Head } else { Method::Get };
headers.retain(|(k, _)| !k.eq_ignore_ascii_case("content-type"));
body = Body::Empty;
} else {
body = replay.ok_or_else(|| NetError::Redirect("a streamed upload can't be resent to the redirect target".into()))?;
}
if next.origin() != origin {
headers.retain(|(k, _)| {
let k = k.to_ascii_lowercase();
!SENSITIVE.contains(&k.as_str()) && !secret_headers.contains(&k)
});
}
url = next;
continue;
}
return Response::new(head, r, url, method, client.config.max_body);
}
}
}
/// A response: status and headers, and the body as a stream ([`Read`]).
pub struct Response {
pub status: u16,
pub reason: String,
pub headers: Vec<(String, String)>,
/// Where the response came from (after redirects).
pub url: Url,
body: Box<dyn Read + Send>,
max_body: u64,
}
impl std::fmt::Debug for Response {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
// headers can carry cookies: only their names
let names: Vec<&str> = self.headers.iter().map(|(k, _)| k.as_str()).collect();
f.debug_struct("Response").field("status", &self.status).field("url", &self.url.origin()).field("headers", &names).finish()
}
}
impl Response {
fn new(head: http::Head, r: BufReader<conn::Conn>, url: Url, method: Method, max_body: u64) -> Result<Response, NetError> {
let framing = Framing::of(&head, method == Method::Head)?;
let raw = BodyReader::new(r, framing);
let gzip = framing != Framing::Empty && head.header("content-encoding").is_some_and(|e| e.trim().eq_ignore_ascii_case("gzip"));
let body: Box<dyn Read + Send> = if gzip { Box::new(flate2::read::GzDecoder::new(raw)) } else { Box::new(raw) };
Ok(Response { status: head.status, reason: head.reason, headers: head.headers, url, body, max_body })
}
/// First value of a header (case-insensitive).
pub fn header(&self, name: &str) -> Option<&str> {
self.headers.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str())
}
pub fn is_success(&self) -> bool {
(200..300).contains(&self.status)
}
/// `Err(Status)` for a non-2xx response.
pub fn error_for_status(self) -> Result<Response, NetError> {
if self.is_success() { Ok(self) } else { Err(NetError::Status { code: self.status, reason: self.reason.clone() }) }
}
/// The whole body (at most the client's `max_body`).
pub fn bytes(mut self) -> Result<Vec<u8>, NetError> {
let cap = self.max_body;
let mut out = Vec::new();
(&mut self.body).take(cap.saturating_add(1)).read_to_end(&mut out)?;
if out.len() as u64 > cap {
return Err(NetError::TooLarge(cap));
}
Ok(out)
}
pub fn text(self) -> Result<String, NetError> {
Ok(String::from_utf8_lossy(&self.bytes()?).into_owned())
}
pub fn json<T: DeserializeOwned>(self) -> Result<T, NetError> {
serde_json::from_slice(&self.bytes()?).map_err(|e| NetError::Json(e.to_string()))
}
}
impl Read for Response {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
self.body.read(buf)
}
}
crates/net/src/conn.rs
+192
View File
@@ -0,0 +1,192 @@
//! One connection: TCP (with connect and stall timeouts and a cancel flag), optionally through an
//! HTTP proxy, optionally wrapped in TLS.
use std::io::{ErrorKind, Read, Write};
use std::net::{TcpStream, ToSocketAddrs};
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, Instant};
use crate::{NetError, Url};
/// How often a blocked read or write wakes up to look at the cancel flag.
const POLL: Duration = Duration::from_millis(250);
/// A TCP stream whose reads and writes only return data or a real error: they wait through
/// socket timeouts, fail with [`NetError::Stalled`] after `stall` without progress and with
/// [`NetError::Cancelled`] once the flag is raised.
pub(crate) struct TimedTcp {
tcp: TcpStream,
stall: Duration,
cancel: Option<Arc<AtomicBool>>,
}
impl TimedTcp {
fn check_cancel(&self) -> std::io::Result<()> {
match &self.cancel {
Some(c) if c.load(Ordering::Relaxed) => Err(NetError::Cancelled.into()),
_ => Ok(()),
}
}
}
fn is_timeout(e: &std::io::Error) -> bool {
matches!(e.kind(), ErrorKind::WouldBlock | ErrorKind::TimedOut | ErrorKind::Interrupted)
}
impl Read for TimedTcp {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let start = Instant::now();
loop {
self.check_cancel()?;
match self.tcp.read(buf) {
Err(e) if is_timeout(&e) => {
if start.elapsed() >= self.stall {
return Err(NetError::Stalled.into());
}
}
other => return other,
}
}
}
}
impl Write for TimedTcp {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
let start = Instant::now();
loop {
self.check_cancel()?;
match self.tcp.write(buf) {
Err(e) if is_timeout(&e) => {
if start.elapsed() >= self.stall {
return Err(NetError::Stalled.into());
}
}
other => return other,
}
}
}
fn flush(&mut self) -> std::io::Result<()> {
self.tcp.flush()
}
}
/// A plain or TLS connection.
pub(crate) enum Conn {
Plain(TimedTcp),
Tls(Box<rustls::StreamOwned<rustls::ClientConnection, TimedTcp>>),
}
impl Read for Conn {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
match self {
Conn::Plain(s) => s.read(buf),
Conn::Tls(s) => match s.read(buf) {
// servers that close without close_notify: treat as end of stream (the body
// framing, Content-Length or chunked, still catches truncation)
Err(e) if e.kind() == ErrorKind::UnexpectedEof => Ok(0),
other => other,
},
}
}
}
impl Write for Conn {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
match self {
Conn::Plain(s) => s.write(buf),
Conn::Tls(s) => s.write(buf),
}
}
fn flush(&mut self) -> std::io::Result<()> {
match self {
Conn::Plain(s) => s.flush(),
Conn::Tls(s) => s.flush(),
}
}
}
pub(crate) struct ConnectOptions<'a> {
pub connect_timeout: Duration,
pub stall: Duration,
pub cancel: Option<Arc<AtomicBool>>,
pub proxy: Option<&'a Url>,
pub tls: Option<Arc<rustls::ClientConfig>>,
}
fn tcp_connect(host: &str, port: u16, timeout: Duration, cancel: &Option<Arc<AtomicBool>>) -> Result<TcpStream, NetError> {
let addrs: Vec<_> = (host, port).to_socket_addrs().map_err(|e| NetError::Connect(format!("{host}: {e}")))?.collect();
let mut last = format!("{host}: no address");
for addr in addrs {
if cancel.as_ref().is_some_and(|c| c.load(Ordering::Relaxed)) {
return Err(NetError::Cancelled);
}
match TcpStream::connect_timeout(&addr, timeout) {
Ok(s) => return Ok(s),
Err(e) => last = format!("{host} ({addr}): {e}"),
}
}
Err(NetError::Connect(last))
}
/// Open a connection to `url`'s origin. Through a proxy, `http://` targets talk to the proxy
/// directly (the request line then carries the absolute URL) and `https://` targets are tunnelled
/// with `CONNECT`.
pub(crate) fn connect(url: &Url, opts: ConnectOptions<'_>) -> Result<Conn, NetError> {
let (host, port) = match opts.proxy {
Some(p) => (p.host.as_str(), p.port),
None => (url.host.as_str(), url.port),
};
let tcp = tcp_connect(host, port, opts.connect_timeout, &opts.cancel)?;
let _ = tcp.set_nodelay(true);
tcp.set_read_timeout(Some(POLL)).map_err(|e| NetError::Io(e.to_string()))?;
tcp.set_write_timeout(Some(POLL)).map_err(|e| NetError::Io(e.to_string()))?;
let mut timed = TimedTcp { tcp, stall: opts.stall, cancel: opts.cancel };
if opts.proxy.is_some() && url.tls {
tunnel(&mut timed, url)?;
}
let Some(config) = opts.tls.filter(|_| url.tls) else {
return Ok(Conn::Plain(timed));
};
let name = rustls::pki_types::ServerName::try_from(url.host.clone()).map_err(|e| NetError::Tls(format!("{}: {e}", url.host)))?;
let mut conn = rustls::ClientConnection::new(config, name).map_err(|e| NetError::Tls(e.to_string()))?;
// finish the handshake now so certificate errors surface as such
while conn.is_handshaking() {
conn.complete_io(&mut timed).map_err(tls_error)?;
}
Ok(Conn::Tls(Box::new(rustls::StreamOwned::new(conn, timed))))
}
/// Map a handshake I/O error, unwrapping our own errors (untrusted certificate, stall, cancel).
fn tls_error(e: std::io::Error) -> NetError {
if let Some(inner) = e.get_ref() {
if let Some(n) = inner.downcast_ref::<NetError>() {
return n.clone();
}
if let Some(rustls::Error::InvalidCertificate(rustls::CertificateError::Other(o))) = inner.downcast_ref::<rustls::Error>()
&& let Some(n) = o.0.downcast_ref::<NetError>()
{
return n.clone();
}
if let Some(rustls::Error::Other(o)) = inner.downcast_ref::<rustls::Error>()
&& let Some(n) = o.0.downcast_ref::<NetError>()
{
return n.clone();
}
}
NetError::Tls(e.to_string())
}
/// `CONNECT host:port` through the proxy; any 2xx opens the tunnel.
fn tunnel(s: &mut TimedTcp, url: &Url) -> Result<(), NetError> {
let target = if url.host.contains(':') { format!("[{}]:{}", url.host, url.port) } else { format!("{}:{}", url.host, url.port) };
let req = format!("CONNECT {target} HTTP/1.1\r\nHost: {target}\r\nUser-Agent: {}\r\n\r\n", crate::user_agent());
s.write_all(req.as_bytes())?;
let head = crate::http::read_head(s)?;
if !(200..300).contains(&head.status) {
return Err(NetError::Connect(format!("proxy refused the tunnel: {} {}", head.status, head.reason)));
}
Ok(())
}
crates/net/src/download.rs
+195
View File
@@ -0,0 +1,195 @@
//! Streaming a download to a file: resumable (`<dest>.part` plus `Range`), size-capped, and
//! checked against an expected size and SHA-256 before it is renamed into place. A file that
//! fails its check is deleted, never kept under its real name.
use std::fs::{File, OpenOptions};
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use sha2::{Digest, Sha256};
use crate::{Client, NetError};
/// What [`Client::download`] checks and reports.
#[derive(Default)]
pub struct DownloadOptions<'a> {
/// Exact size in bytes, when known.
pub size: Option<u64>,
/// SHA-256 (hex, any case), when known.
pub sha256: Option<String>,
/// Refuse anything larger (0: no cap besides `size`).
pub max_size: u64,
/// Extra headers (`secret`: dropped on cross-origin redirects).
pub headers: Vec<(String, String, bool)>,
pub cancel: Option<Arc<AtomicBool>>,
/// `(bytes so far, total when known)`.
pub progress: Option<&'a mut dyn FnMut(u64, Option<u64>)>,
}
/// A finished download.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Downloaded {
pub bytes: u64,
/// Lower-case hex SHA-256 of the file.
pub sha256: String,
}
fn part_path(dest: &Path) -> PathBuf {
let mut name = dest.file_name().map(|n| n.to_os_string()).unwrap_or_default();
name.push(".part");
dest.with_file_name(name)
}
fn io(path: &Path, e: std::io::Error) -> NetError {
NetError::Io(format!("{}: {e}", path.display()))
}
/// Parse `Content-Range: bytes START-END/TOTAL` → (start, total).
fn content_range(v: &str) -> Option<(u64, Option<u64>)> {
let rest = v.trim().strip_prefix("bytes")?.trim();
let (range, total) = rest.split_once('/')?;
let start = range.split_once('-')?.0.trim().parse().ok()?;
Some((start, total.trim().parse().ok()))
}
impl Client {
/// Download `url` to `dest`, resuming a previous `<dest>.part`.
pub fn download(&self, url: &str, dest: &Path, mut opts: DownloadOptions<'_>) -> Result<Downloaded, NetError> {
let part = part_path(dest);
let cap = match (opts.size, opts.max_size) {
(Some(s), _) => s,
(None, 0) => u64::MAX,
(None, m) => m,
};
// hash what is already there
let mut hasher = Sha256::new();
let mut have = 0u64;
if let Ok(mut f) = File::open(&part) {
let mut buf = vec![0u8; 256 * 1024];
loop {
let n = f.read(&mut buf).map_err(|e| io(&part, e))?;
if n == 0 {
break;
}
hasher.update(buf.get(..n).unwrap_or_default());
have = have.saturating_add(n as u64);
}
}
if have > cap {
// left over from something else: start again
have = 0;
hasher = Sha256::new();
}
let mut req = self.get(url).identity_encoding();
for (k, v, secret) in &opts.headers {
req = if *secret { req.secret_header(k, v) } else { req.header(k, v) };
}
if let Some(c) = &opts.cancel {
req = req.cancel(c.clone());
}
let done_already = opts.size.is_some_and(|s| s == have && s > 0);
if have > 0 && !done_already {
req = req.header("Range", &format!("bytes={have}-"));
}
let (mut resp, mut file) = if done_already {
(None, None)
} else {
let resp = req.send()?;
let append = match resp.status {
206 if have > 0 => match resp.header("content-range").and_then(content_range) {
Some((start, _)) if start == have => true,
_ => return Err(NetError::Protocol("server resumed at the wrong offset".into())),
},
200 => false,
416 if have > 0 => {
let _ = std::fs::remove_file(&part);
return Err(NetError::Protocol("server refused to resume; the partial file was removed, try again".into()));
}
code => return Err(NetError::Status { code, reason: resp.reason.clone() }),
};
if !append {
have = 0;
hasher = Sha256::new();
}
if let Some(parent) = part.parent().filter(|p| !p.as_os_str().is_empty()) {
std::fs::create_dir_all(parent).map_err(|e| io(parent, e))?;
}
let file = OpenOptions::new().create(true).write(true).append(append).truncate(!append).open(&part).map_err(|e| io(&part, e))?;
(Some(resp), Some(file))
};
let total = opts.size.or_else(|| {
let r = resp.as_ref()?;
let len: u64 = r.header("content-length")?.trim().parse().ok()?;
Some(len.saturating_add(have))
});
if let (Some(t), true) = (total, cap != u64::MAX)
&& t > cap
{
return Err(NetError::TooLarge(cap));
}
let mut too_large = false;
if let (Some(resp), Some(file)) = (resp.as_mut(), file.as_mut()) {
let mut buf = vec![0u8; 256 * 1024];
loop {
let n = resp.read(&mut buf)?;
if n == 0 {
break;
}
let chunk = buf.get(..n).unwrap_or_default();
have = have.saturating_add(n as u64);
if have > cap {
too_large = true;
break;
}
file.write_all(chunk).map_err(|e| io(&part, e))?;
hasher.update(chunk);
if let Some(p) = opts.progress.as_mut() {
p(have, total);
}
}
if !too_large {
file.sync_all().map_err(|e| io(&part, e))?;
}
}
drop(file);
if too_large {
let _ = std::fs::remove_file(&part);
return Err(NetError::TooLarge(cap));
}
let sha = hasher.finalize().iter().map(|b| format!("{b:02x}")).collect::<String>();
let fail = |why: String| {
let _ = std::fs::remove_file(&part);
Err(NetError::Verify(why))
};
if let Some(size) = opts.size
&& size != have
{
return fail(format!("expected {size} bytes, got {have}"));
}
if let Some(want) = &opts.sha256
&& !want.trim().eq_ignore_ascii_case(&sha)
{
return fail("SHA-256 does not match".into());
}
std::fs::rename(&part, dest).map_err(|e| io(dest, e))?;
Ok(Downloaded { bytes: have, sha256: sha })
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_content_range() {
assert_eq!(content_range("bytes 10-99/100"), Some((10, Some(100))));
assert_eq!(content_range("bytes 10-99/*"), Some((10, None)));
assert_eq!(content_range("items 1-2/3"), None);
assert_eq!(part_path(Path::new("/a/b.bin")), PathBuf::from("/a/b.bin.part"));
}
}
crates/net/src/error.rs
+78
View File
@@ -0,0 +1,78 @@
//! The one error type of the crate.
/// Why a request failed. Messages never contain header values, bodies or credentials.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum NetError {
/// The URL can't be used (scheme, host, characters).
BadUrl(String),
/// A header name or value can't be sent (line breaks, colons in names).
BadHeader(String),
/// Name resolution or connecting failed, or took longer than the connect timeout.
Connect(String),
/// No data moved for the stall timeout.
Stalled,
/// The cancel flag was raised.
Cancelled,
/// TLS failed (handshake, protocol).
Tls(String),
/// The server's certificate is not trusted and nobody accepted it (no trust-on-first-use hook,
/// or the hook declined). Carries the SHA-256 fingerprint so the UI can show it.
UntrustedCertificate { host: String, fingerprint: String },
/// The server's response could not be understood.
Protocol(String),
/// More redirects than allowed, or a redirect that a non-replayable body can't follow.
Redirect(String),
/// The response (or a file) is larger than the allowed size.
TooLarge(u64),
/// The server answered with an error status ([`crate::Response::error_for_status`]).
Status { code: u16, reason: String },
/// A JSON body could not be written or read.
Json(String),
/// A downloaded file does not match its expected size or SHA-256.
Verify(String),
/// Any other I/O error (local files included).
Io(String),
}
impl std::fmt::Display for NetError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
NetError::BadUrl(e) => write!(f, "unusable URL: {e}"),
NetError::BadHeader(e) => write!(f, "unusable header: {e}"),
NetError::Connect(e) => write!(f, "could not connect: {e}"),
NetError::Stalled => write!(f, "the server stopped responding"),
NetError::Cancelled => write!(f, "cancelled"),
NetError::Tls(e) => write!(f, "secure connection failed: {e}"),
NetError::UntrustedCertificate { host, fingerprint } => {
write!(f, "the certificate of {host} is not trusted (SHA-256 fingerprint {fingerprint})")
}
NetError::Protocol(e) => write!(f, "unexpected response: {e}"),
NetError::Redirect(e) => write!(f, "redirect not followed: {e}"),
NetError::TooLarge(n) => write!(f, "larger than the allowed {n} bytes"),
NetError::Status { code, reason } => write!(f, "server answered {code} {reason}"),
NetError::Json(e) => write!(f, "JSON: {e}"),
NetError::Verify(e) => write!(f, "verification failed: {e}"),
NetError::Io(e) => write!(f, "{e}"),
}
}
}
impl std::error::Error for NetError {}
impl From<std::io::Error> for NetError {
fn from(e: std::io::Error) -> Self {
// the stream wrappers carry our own errors through io::Error (rustls, flate2)
if let Some(inner) = e.get_ref().and_then(|i| i.downcast_ref::<NetError>()) {
return inner.clone();
}
NetError::Io(e.to_string())
}
}
impl From<NetError> for std::io::Error {
fn from(e: NetError) -> Self {
// never `Interrupted`: std retries those, which would swallow a cancel
let kind = if e == NetError::Stalled { std::io::ErrorKind::TimedOut } else { std::io::ErrorKind::Other };
std::io::Error::new(kind, e)
}
}
crates/net/src/http.rs
+246
View File
@@ -0,0 +1,246 @@
//! HTTP/1.1 wire format: the response head and body framing (Content-Length, chunked, or read
//! until close). Everything read from the server is bounded.
use std::io::{BufRead, Read};
use crate::NetError;
/// Largest response head (status line plus headers).
const MAX_HEAD: usize = 64 * 1024;
/// Most header lines.
const MAX_HEADERS: usize = 256;
/// Longest chunk-size or trailer line.
const MAX_LINE: usize = 8 * 1024;
/// A response's status line and headers.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Head {
pub status: u16,
pub reason: String,
/// Header names as sent; look them up with [`Head::header`].
pub headers: Vec<(String, String)>,
}
impl Head {
/// First value of a header (case-insensitive name).
pub fn header(&self, name: &str) -> Option<&str> {
self.headers.iter().find(|(k, _)| k.eq_ignore_ascii_case(name)).map(|(_, v)| v.as_str())
}
}
/// Read one line ending in LF (CR stripped), at most `max` bytes, one byte at a time (so nothing
/// past the line is consumed: a proxy tunnel's TLS bytes follow its head).
fn read_line(r: &mut impl Read, max: usize) -> Result<String, NetError> {
let mut line = Vec::new();
let mut b = [0u8; 1];
loop {
let n = r.read(&mut b)?;
if n == 0 {
return Err(NetError::Protocol("connection closed in the middle of a line".into()));
}
if b[0] == b'\n' {
break;
}
if line.len() >= max {
return Err(NetError::Protocol("line too long".into()));
}
line.push(b[0]);
}
if line.last() == Some(&b'\r') {
line.pop();
}
Ok(String::from_utf8_lossy(&line).into_owned())
}
/// Read a status line and headers. `1xx` interim responses are skipped.
pub fn read_head(r: &mut impl Read) -> Result<Head, NetError> {
loop {
let mut used = 0usize;
let status_line = read_line(r, MAX_HEAD)?;
used += status_line.len();
let mut parts = status_line.splitn(3, ' ');
let version = parts.next().unwrap_or_default();
if !version.starts_with("HTTP/1.") {
return Err(NetError::Protocol(format!("not an HTTP/1.x response: {:?}", status_line.chars().take(40).collect::<String>())));
}
let status: u16 = parts
.next()
.and_then(|s| s.parse().ok())
.filter(|s| (100..1000).contains(s))
.ok_or_else(|| NetError::Protocol("bad status code".into()))?;
let reason = parts.next().unwrap_or_default().to_string();
let mut headers = Vec::new();
loop {
let line = read_line(r, MAX_HEAD)?;
used += line.len() + 2;
if used > MAX_HEAD {
return Err(NetError::Protocol("response head larger than 64 KiB".into()));
}
if line.is_empty() {
break;
}
if headers.len() >= MAX_HEADERS {
return Err(NetError::Protocol("too many headers".into()));
}
let Some((k, v)) = line.split_once(':') else {
return Err(NetError::Protocol("malformed header line".into()));
};
headers.push((k.trim().to_string(), v.trim().to_string()));
}
if (100..200).contains(&status) && status != 101 {
continue;
}
return Ok(Head { status, reason, headers });
}
}
/// How the body is delimited.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum Framing {
Empty,
Length(u64),
Chunked,
Close,
}
impl Framing {
pub(crate) fn of(head: &Head, request_was_head: bool) -> Result<Framing, NetError> {
if request_was_head || head.status == 204 || head.status == 304 || (100..200).contains(&head.status) {
return Ok(Framing::Empty);
}
if head.header("transfer-encoding").is_some_and(|t| t.to_ascii_lowercase().contains("chunked")) {
return Ok(Framing::Chunked);
}
match head.header("content-length") {
Some(v) => v.trim().parse().map(Framing::Length).map_err(|_| NetError::Protocol("bad Content-Length".into())),
None => Ok(Framing::Close),
}
}
}
/// Reads exactly the body, per its framing.
pub(crate) struct BodyReader<R> {
inner: R,
state: State,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum State {
/// Bytes left (Content-Length, or of the current chunk).
Left {
left: u64,
chunked: bool,
},
/// Before a chunk-size line (`first`: no CRLF of a previous chunk to consume).
ChunkStart {
first: bool,
},
UntilClose,
Done,
}
impl<R: BufRead> BodyReader<R> {
pub(crate) fn new(inner: R, framing: Framing) -> Self {
let state = match framing {
Framing::Empty | Framing::Length(0) => State::Done,
Framing::Length(n) => State::Left { left: n, chunked: false },
Framing::Chunked => State::ChunkStart { first: true },
Framing::Close => State::UntilClose,
};
BodyReader { inner, state }
}
fn next_chunk(&mut self, first: bool) -> Result<(), NetError> {
if !first {
let crlf = read_line(&mut self.inner, 2)?;
if !crlf.is_empty() {
return Err(NetError::Protocol("missing CRLF after chunk".into()));
}
}
let line = read_line(&mut self.inner, MAX_LINE)?;
let size = line.split(';').next().unwrap_or_default().trim();
let size = u64::from_str_radix(size, 16).map_err(|_| NetError::Protocol("bad chunk size".into()))?;
if size == 0 {
// trailers, then the empty line
let mut used = 0usize;
loop {
let t = read_line(&mut self.inner, MAX_LINE)?;
if t.is_empty() {
break;
}
used += t.len();
if used > MAX_HEAD {
return Err(NetError::Protocol("trailers too large".into()));
}
}
self.state = State::Done;
} else {
self.state = State::Left { left: size, chunked: true };
}
Ok(())
}
}
impl<R: BufRead> Read for BodyReader<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
if buf.is_empty() {
return Ok(0);
}
loop {
match self.state {
State::Done => return Ok(0),
State::UntilClose => return self.inner.read(buf),
State::ChunkStart { first } => self.next_chunk(first)?,
State::Left { left, chunked } => {
let want = usize::try_from(left).unwrap_or(usize::MAX).min(buf.len());
let n = self.inner.read(buf.get_mut(..want).unwrap_or_default())?;
if n == 0 {
return Err(NetError::Protocol("connection closed before the end of the body".into()).into());
}
let left = left.saturating_sub(n as u64);
self.state = match (left, chunked) {
(0, true) => State::ChunkStart { first: false },
(0, false) => State::Done,
(left, chunked) => State::Left { left, chunked },
};
return Ok(n);
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
fn body(raw: &[u8], framing: Framing) -> Result<Vec<u8>, NetError> {
let mut out = Vec::new();
BodyReader::new(Cursor::new(raw.to_vec()), framing).read_to_end(&mut out)?;
Ok(out)
}
#[test]
fn heads_and_bodies() {
let mut r = Cursor::new(b"HTTP/1.1 100 Continue\r\n\r\nHTTP/1.1 200 OK\r\nContent-Length: 3\r\nX-A: b \r\n\r\nabc".to_vec());
let h = read_head(&mut r).unwrap();
assert_eq!((h.status, h.reason.as_str(), h.header("x-a")), (200, "OK", Some("b")));
assert_eq!(Framing::of(&h, false).unwrap(), Framing::Length(3));
assert_eq!(body(b"4;x=y\r\nWiki\r\n5\r\npedia\r\n0\r\nT: v\r\n\r\n", Framing::Chunked).unwrap(), b"Wikipedia");
assert_eq!(body(b"rest", Framing::Close).unwrap(), b"rest");
}
#[test]
fn hostile_responses_are_errors() {
for raw in [&b"SSH-2.0\r\n\r\n"[..], b"HTTP/1.1 abc OK\r\n\r\n", b"HTTP/1.1 200 OK\r\nnocolon\r\n\r\n", b"HTTP/1.1 200"] {
assert!(read_head(&mut Cursor::new(raw.to_vec())).is_err());
}
let huge = format!("HTTP/1.1 200 OK\r\n{}\r\n", "X: y\r\n".repeat(20_000));
assert!(read_head(&mut Cursor::new(huge.into_bytes())).is_err());
assert!(body(b"zz\r\n", Framing::Chunked).is_err());
assert!(body(b"ffffffffffffffffffff\r\n", Framing::Chunked).is_err());
assert!(body(b"3\r\nabcX", Framing::Chunked).is_err());
assert!(body(b"ab", Framing::Length(5)).is_err());
}
}
crates/net/src/lib.rs
+48
View File
@@ -0,0 +1,48 @@
//! HTTP(S) for everything that talks to a server: Immich, publish services, map tiles and
//! geocoding, model downloads. Pure Rust: `std::net` sockets and rustls with the RustCrypto
//! provider (no `ring`/`aws-lc-rs`, nothing compiled from C), Mozilla roots from `webpki-roots`,
//! gzip through `flate2`'s Rust backend.
//!
//! - [`Client`] with [`ClientConfig`]: connect and stall timeouts, redirects (credentials never
//! follow a redirect to another origin, and https never downgrades to http), an optional HTTP
//! proxy (`CONNECT` for https), and per-connection [`Trust`]: extra CA certificates, pinned
//! fingerprints, and trust-on-first-use with a confirmation hook for self-signed home servers.
//! - [`Request`]: GET/HEAD/POST/PUT/PATCH/DELETE, headers ([`Request::secret_header`] /
//! [`Request::bearer`] for credentials), JSON or raw bodies, streaming [`Multipart`] uploads
//! with progress, cancellation through an `AtomicBool`.
//! - [`Response`]: status, headers, the body as a [`std::io::Read`] stream (gzip decoded), or
//! capped [`Response::bytes`] / [`Response::text`] / [`Response::json`].
//! - [`Client::download`]: resumable, size-capped, SHA-256-verified downloads to a file.
//!
//! Blocking by design: callers run requests on worker threads. Logging never includes header
//! values, bodies or query strings (they carry API keys and share keys).
//!
//! Native only: on wasm32 this crate is empty.
#![cfg(not(target_arch = "wasm32"))]
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
mod client;
mod conn;
mod download;
mod error;
pub mod http;
mod multipart;
mod tls;
mod url;
pub use client::{Client, ClientConfig, Method, Progress, Request, Response};
pub use download::{DownloadOptions, Downloaded};
pub use error::NetError;
pub use multipart::Multipart;
pub use tls::{CertInfo, Fingerprint, TofuHook, Trust};
pub use url::Url;
/// `User-Agent`: the brand's binary name and the version.
pub fn user_agent() -> String {
format!("{}/{}", dac_brand::BINARY, env!("CARGO_PKG_VERSION"))
}
#[cfg(test)]
mod tests;
crates/net/src/multipart.rs
+214
View File
@@ -0,0 +1,214 @@
//! `multipart/form-data` bodies that stream: file parts are read from disk (or any reader) while
//! sending, never loaded whole, and the total length is known up front so the request carries a
//! `Content-Length` and progress has a denominator.
use std::fs::File;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use sha2::{Digest, Sha256};
use crate::NetError;
enum Data {
Bytes(Vec<u8>),
File(PathBuf, u64),
Reader(Box<dyn Read + Send>, u64),
}
struct Part {
head: String,
data: Data,
}
impl Part {
fn len(&self) -> u64 {
match &self.data {
Data::Bytes(b) => b.len() as u64,
Data::File(_, n) | Data::Reader(_, n) => *n,
}
}
}
/// A `multipart/form-data` body under construction.
pub struct Multipart {
boundary: String,
parts: Vec<Part>,
}
impl std::fmt::Debug for Multipart {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Multipart").field("parts", &self.parts.len()).field("len", &self.len()).finish()
}
}
impl Default for Multipart {
fn default() -> Self {
Multipart::new()
}
}
/// Quote a form field or file name: no line breaks, quotes escaped.
fn quoted(s: &str) -> String {
s.chars().filter(|c| *c != '\r' && *c != '\n').map(|c| if c == '"' { "%22".to_string() } else { c.to_string() }).collect()
}
fn clean(s: &str) -> String {
s.chars().filter(|c| !c.is_control()).collect()
}
impl Multipart {
pub fn new() -> Multipart {
static COUNTER: AtomicU64 = AtomicU64::new(0);
let mut h = Sha256::new();
h.update(format!("{:?}{}{:?}", std::time::SystemTime::now(), std::process::id(), std::thread::current().id()));
h.update(COUNTER.fetch_add(1, Ordering::Relaxed).to_le_bytes());
let hex: String = h.finalize().iter().take(16).map(|b| format!("{b:02x}")).collect();
Multipart { boundary: format!("----dac-{hex}"), parts: Vec::new() }
}
fn push(&mut self, name: &str, file: Option<(&str, &str)>, data: Data) {
let mut head = format!("--{}\r\nContent-Disposition: form-data; name=\"{}\"", self.boundary, quoted(name));
if let Some((filename, content_type)) = file {
head.push_str(&format!("; filename=\"{}\"\r\nContent-Type: {}", quoted(filename), clean(content_type)));
}
head.push_str("\r\n\r\n");
self.parts.push(Part { head, data });
}
/// A text field.
pub fn text(mut self, name: &str, value: &str) -> Multipart {
self.push(name, None, Data::Bytes(value.as_bytes().to_vec()));
self
}
/// A file part from memory.
pub fn bytes(mut self, name: &str, filename: &str, content_type: &str, data: Vec<u8>) -> Multipart {
self.push(name, Some((filename, content_type)), Data::Bytes(data));
self
}
/// A file part streamed from disk (its size is taken now; a file that changes size while
/// uploading fails the request rather than sending a corrupt body).
pub fn file(mut self, name: &str, filename: &str, content_type: &str, path: &Path) -> Result<Multipart, NetError> {
let len = std::fs::metadata(path).map_err(|e| NetError::Io(format!("{}: {e}", path.display())))?.len();
self.push(name, Some((filename, content_type)), Data::File(path.to_path_buf(), len));
Ok(self)
}
/// A file part streamed from a reader that yields exactly `len` bytes.
pub fn reader(mut self, name: &str, filename: &str, content_type: &str, reader: Box<dyn Read + Send>, len: u64) -> Multipart {
self.push(name, Some((filename, content_type)), Data::Reader(reader, len));
self
}
/// The `Content-Type` header value.
pub fn content_type(&self) -> String {
format!("multipart/form-data; boundary={}", self.boundary)
}
fn closing(&self) -> String {
format!("--{}--\r\n", self.boundary)
}
/// Exact body length in bytes.
pub fn len(&self) -> u64 {
let parts: u64 = self.parts.iter().map(|p| (p.head.len() as u64).saturating_add(p.len()).saturating_add(2)).fold(0, u64::saturating_add);
parts.saturating_add(self.closing().len() as u64)
}
pub fn is_empty(&self) -> bool {
self.parts.is_empty()
}
/// Write the body, calling `progress(sent)` as it goes.
pub(crate) fn write_to(self, w: &mut impl Write, progress: &mut dyn FnMut(u64)) -> Result<(), NetError> {
let closing = self.closing();
let mut sent = 0u64;
let mut buf = vec![0u8; 64 * 1024];
for part in self.parts {
w.write_all(part.head.as_bytes())?;
sent = sent.saturating_add(part.head.len() as u64);
let expected = part.len();
match part.data {
Data::Bytes(b) => {
for chunk in b.chunks(64 * 1024) {
w.write_all(chunk)?;
sent = sent.saturating_add(chunk.len() as u64);
progress(sent);
}
}
Data::File(path, _) => {
let f = File::open(&path).map_err(|e| NetError::Io(format!("{}: {e}", path.display())))?;
sent = copy_exact(f, expected, w, &mut buf, sent, progress)?;
}
Data::Reader(r, _) => sent = copy_exact(r, expected, w, &mut buf, sent, progress)?,
}
w.write_all(b"\r\n")?;
sent = sent.saturating_add(2);
}
w.write_all(closing.as_bytes())?;
progress(sent.saturating_add(closing.len() as u64));
Ok(())
}
}
/// Copy exactly `len` bytes from `r` (fewer or more is an error: the Content-Length is promised).
fn copy_exact(mut r: impl Read, len: u64, w: &mut impl Write, buf: &mut [u8], mut sent: u64, progress: &mut dyn FnMut(u64)) -> Result<u64, NetError> {
let mut left = len;
while left > 0 {
let want = usize::try_from(left).unwrap_or(usize::MAX).min(buf.len());
let chunk = buf.get_mut(..want).unwrap_or_default();
let n = r.read(chunk).map_err(|e| NetError::Io(e.to_string()))?;
if n == 0 {
return Err(NetError::Io("upload source ended early (file changed while uploading?)".into()));
}
w.write_all(chunk.get(..n).unwrap_or_default())?;
left = left.saturating_sub(n as u64);
sent = sent.saturating_add(n as u64);
progress(sent);
}
let mut probe = [0u8; 1];
if r.read(&mut probe).map_err(|e| NetError::Io(e.to_string()))? != 0 {
return Err(NetError::Io("upload source is longer than announced (file changed while uploading?)".into()));
}
Ok(sent)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn length_matches_bytes_written() {
let m = Multipart::new().text("deviceId", "x").bytes("a\"b", "f\r\n.jpg", "image/jpeg", vec![7; 100_000]).reader(
"r",
"r.bin",
"application/octet-stream",
Box::new(std::io::Cursor::new(vec![1u8; 10])),
10,
);
let len = m.len();
let mut out = Vec::new();
let mut last = 0;
m.write_to(&mut out, &mut |s| last = s).unwrap();
assert_eq!(out.len() as u64, len);
assert_eq!(last, len);
let text = String::from_utf8_lossy(&out);
assert!(text.contains("name=\"a%22b\"; filename=\"f.jpg\""));
}
#[test]
fn short_or_long_reader_is_an_error() {
let m = Multipart::new().reader("r", "r", "x/y", Box::new(std::io::Cursor::new(vec![1u8; 5])), 10);
assert!(m.write_to(&mut Vec::new(), &mut |_| {}).is_err());
let m = Multipart::new().reader("r", "r", "x/y", Box::new(std::io::Cursor::new(vec![1u8; 15])), 10);
assert!(m.write_to(&mut Vec::new(), &mut |_| {}).is_err());
}
#[test]
fn boundaries_differ() {
assert_ne!(Multipart::new().boundary, Multipart::new().boundary);
}
}
crates/net/src/tests.rs
+441
View File
@@ -0,0 +1,441 @@
//! End-to-end tests against in-process servers on 127.0.0.1 (no external network).
use std::io::{BufRead, BufReader, Read, Write};
use std::net::TcpListener;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex};
use std::thread::JoinHandle;
use std::time::Duration;
use super::*;
#[derive(Debug, Clone)]
struct Req {
method: String,
target: String,
headers: Vec<(String, String)>,
body: Vec<u8>,
}
impl Req {
fn header(&self, n: &str) -> Option<&str> {
self.headers.iter().find(|(k, _)| k.eq_ignore_ascii_case(n)).map(|(_, v)| v.as_str())
}
}
fn read_req(s: impl Read) -> Option<Req> {
let mut r = BufReader::new(s);
let mut line = String::new();
r.read_line(&mut line).ok()?;
let mut it = line.split_whitespace();
let (method, target) = (it.next()?.to_string(), it.next()?.to_string());
let mut headers = Vec::new();
loop {
let mut l = String::new();
r.read_line(&mut l).ok()?;
let l = l.trim_end();
if l.is_empty() {
break;
}
let (k, v) = l.split_once(':')?;
headers.push((k.trim().to_string(), v.trim().to_string()));
}
let len: usize = headers.iter().find(|(k, _)| k.eq_ignore_ascii_case("content-length")).and_then(|(_, v)| v.parse().ok()).unwrap_or(0);
let mut body = vec![0; len];
r.read_exact(&mut body).ok()?;
Some(Req { method, target, headers, body })
}
fn response(status: &str, headers: &[(&str, &str)], body: &[u8]) -> Vec<u8> {
let mut out = format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\n", body.len());
for (k, v) in headers {
out.push_str(&format!("{k}: {v}\r\n"));
}
out.push_str("\r\n");
let mut out = out.into_bytes();
out.extend_from_slice(body);
out
}
type Handler = Arc<dyn Fn(Req) -> Vec<u8> + Send + Sync>;
/// Serve `n` connections with `handler`; the requests seen are collected.
fn serve(n: usize, handler: impl Fn(Req) -> Vec<u8> + Send + Sync + 'static) -> (u16, Arc<Mutex<Vec<Req>>>, JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
let seen = Arc::new(Mutex::new(Vec::new()));
let seen2 = seen.clone();
let handler: Handler = Arc::new(handler);
let h = std::thread::spawn(move || {
for _ in 0..n {
let Ok((mut s, _)) = listener.accept() else { return };
let Some(req) = read_req(&mut s) else { continue };
seen2.lock().unwrap().push(req.clone());
let _ = s.write_all(&handler(req));
}
});
(port, seen, h)
}
fn client() -> Client {
Client::new(ClientConfig { stall_timeout: Duration::from_secs(5), ..ClientConfig::default() }).unwrap()
}
#[test]
fn get_json_gzip_and_user_agent() {
let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
gz.write_all(br#"{"version":"1.2.3"}"#).unwrap();
let gz = gz.finish().unwrap();
let (port, seen, h) = serve(1, move |_| response("200 OK", &[("Content-Encoding", "gzip"), ("Content-Type", "application/json")], &gz));
#[derive(serde::Deserialize)]
struct V {
version: String,
}
let v: V = client().get(&format!("http://127.0.0.1:{port}/api/server/version")).send().unwrap().error_for_status().unwrap().json().unwrap();
assert_eq!(v.version, "1.2.3");
h.join().unwrap();
let req = seen.lock().unwrap()[0].clone();
assert_eq!(req.header("user-agent"), Some(user_agent().as_str()));
assert!(user_agent().starts_with(dac_brand::BINARY));
assert_eq!(req.header("accept-encoding"), Some("gzip"));
}
#[test]
fn methods_and_bodies() {
let (port, seen, h) = serve(4, |r| response("200 OK", &[], &r.body));
let c = client();
let base = format!("http://127.0.0.1:{port}/x");
assert_eq!(c.post(&base).json(&serde_json::json!({"a": 1})).send().unwrap().text().unwrap(), r#"{"a":1}"#);
assert_eq!(c.put(&base).body("text/plain", b"put".to_vec()).send().unwrap().bytes().unwrap(), b"put");
assert_eq!(c.patch(&base).json(&[1, 2]).send().unwrap().text().unwrap(), "[1,2]");
let r = c.delete(&base).send().unwrap();
assert!(r.is_success());
h.join().unwrap();
let seen = seen.lock().unwrap();
let methods: Vec<&str> = seen.iter().map(|r| r.method.as_str()).collect();
assert_eq!(methods, ["POST", "PUT", "PATCH", "DELETE"]);
assert_eq!(seen[0].header("content-type"), Some("application/json"));
}
#[test]
fn error_status_and_bad_headers() {
let (port, _, h) = serve(1, |_| response("404 Not Found", &[], b"nope"));
let e = client().get(&format!("http://127.0.0.1:{port}/")).send().unwrap().error_for_status().unwrap_err();
assert_eq!(e, NetError::Status { code: 404, reason: "Not Found".into() });
h.join().unwrap();
let e = client().get("http://127.0.0.1:1/").header("X", "a\r\nInjected: 1").send().unwrap_err();
assert!(matches!(e, NetError::BadHeader(_)));
assert!(!e.to_string().contains("Injected"));
}
#[test]
fn multipart_upload_streams_with_progress() {
let dir = tempdir("mp");
let file = dir.join("photo.jpg");
std::fs::write(&file, vec![0xAB; 300_000]).unwrap();
let (port, seen, h) = serve(1, |r| response("201 Created", &[], format!("{}", r.body.len()).as_bytes()));
let form = Multipart::new().text("deviceAssetId", "abc").file("assetData", "photo.jpg", "image/jpeg", &file).unwrap();
let expected = form.len();
let mut calls = 0u32;
let mut last = (0, None);
let mut progress = |s: u64, t: Option<u64>| {
calls += 1;
last = (s, t);
};
let r = client().post(&format!("http://127.0.0.1:{port}/api/assets")).multipart(form).progress(&mut progress).send().unwrap();
assert_eq!(r.status, 201);
assert_eq!(r.text().unwrap(), expected.to_string());
assert!(calls > 1);
assert_eq!(last, (expected, Some(expected)));
h.join().unwrap();
let req = seen.lock().unwrap()[0].clone();
assert!(req.header("content-type").unwrap().starts_with("multipart/form-data; boundary="));
let body = String::from_utf8_lossy(&req.body);
assert!(body.contains("name=\"assetData\"; filename=\"photo.jpg\""));
let _ = std::fs::remove_dir_all(dir);
}
#[test]
fn redirects_keep_credentials_on_the_origin_only() {
let (other, other_seen, h2) = serve(1, |_| response("200 OK", &[], b"elsewhere"));
let (port, seen, h1) = serve(2, move |r| {
if r.target == "/a" {
response("302 Found", &[("Location", "/b")], b"")
} else {
response("307 Temporary Redirect", &[("Location", &format!("http://localhost:{other}/c"))], b"")
}
});
let r = client().post(&format!("http://127.0.0.1:{port}/a")).bearer("s3cret-token").header("X-Api-Key", "k3y").json(&1).send().unwrap();
assert_eq!(r.text().unwrap(), "elsewhere");
h1.join().unwrap();
h2.join().unwrap();
let seen = seen.lock().unwrap();
assert_eq!(seen[1].method, "GET"); // 302 after POST becomes GET
assert!(seen[1].header("authorization").is_some());
let other_seen = other_seen.lock().unwrap();
assert_eq!(other_seen[0].target, "/c");
assert!(other_seen[0].header("authorization").is_none());
assert!(other_seen[0].header("x-api-key").is_none());
}
#[test]
fn too_many_redirects() {
let (port, _, h) = serve(3, |_| response("302 Found", &[("Location", "/loop")], b""));
let c = Client::new(ClientConfig { max_redirects: 2, ..ClientConfig::default() }).unwrap();
assert!(matches!(c.get(&format!("http://127.0.0.1:{port}/")).send(), Err(NetError::Redirect(_))));
h.join().unwrap();
}
#[test]
fn stall_timeout_and_cancel() {
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
let h = std::thread::spawn(move || {
let mut held = Vec::new();
for _ in 0..2 {
if let Ok((s, _)) = listener.accept() {
held.push(s); // never answer
}
}
std::thread::sleep(Duration::from_secs(3));
});
let c = Client::new(ClientConfig { stall_timeout: Duration::from_millis(600), ..ClientConfig::default() }).unwrap();
let t = std::time::Instant::now();
assert_eq!(c.get(&format!("http://127.0.0.1:{port}/")).send().unwrap_err(), NetError::Stalled);
assert!(t.elapsed() < Duration::from_secs(3));
let c = Client::new(ClientConfig { stall_timeout: Duration::from_secs(60), ..ClientConfig::default() }).unwrap();
let flag = Arc::new(AtomicBool::new(false));
let f2 = flag.clone();
std::thread::spawn(move || {
std::thread::sleep(Duration::from_millis(300));
f2.store(true, Ordering::Relaxed);
});
let t = std::time::Instant::now();
assert_eq!(c.get(&format!("http://127.0.0.1:{port}/")).cancel(flag).send().unwrap_err(), NetError::Cancelled);
assert!(t.elapsed() < Duration::from_secs(5));
h.join().unwrap();
}
#[test]
fn connect_failure_is_an_error() {
let port = TcpListener::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port(); // closed again
assert!(matches!(client().get(&format!("http://127.0.0.1:{port}/")).send(), Err(NetError::Connect(_))));
}
#[test]
fn http_proxy_gets_absolute_form() {
let (port, seen, h) = serve(1, |_| response("200 OK", &[], b"via proxy"));
let c = Client::new(ClientConfig { proxy: Some(format!("http://127.0.0.1:{port}")), ..ClientConfig::default() }).unwrap();
assert_eq!(c.get("http://photos.invalid:2283/api/x?k=v").send().unwrap().text().unwrap(), "via proxy");
h.join().unwrap();
assert_eq!(seen.lock().unwrap()[0].target, "http://photos.invalid:2283/api/x?k=v");
assert!(Client::new(ClientConfig { proxy: Some("https://p".into()), ..ClientConfig::default() }).is_err());
}
#[test]
fn chunked_and_truncated_bodies() {
let (port, _, h) = serve(2, |r| {
if r.target == "/chunked" {
b"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n3\r\nabc\r\n2\r\nde\r\n0\r\n\r\n".to_vec()
} else {
b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nshort".to_vec()
}
});
assert_eq!(client().get(&format!("http://127.0.0.1:{port}/chunked")).send().unwrap().text().unwrap(), "abcde");
assert!(client().get(&format!("http://127.0.0.1:{port}/short")).send().unwrap().bytes().is_err());
h.join().unwrap();
}
#[test]
fn body_cap() {
let (port, _, h) = serve(1, |_| response("200 OK", &[], &[1u8; 5000]));
let c = Client::new(ClientConfig { max_body: 1000, ..ClientConfig::default() }).unwrap();
assert_eq!(c.get(&format!("http://127.0.0.1:{port}/")).send().unwrap().bytes().unwrap_err(), NetError::TooLarge(1000));
h.join().unwrap();
}
fn tempdir(tag: &str) -> std::path::PathBuf {
let d = std::env::temp_dir().join(format!("dac-net-test-{tag}-{}-{:?}", std::process::id(), std::thread::current().id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
fn sha256_hex(data: &[u8]) -> String {
use sha2::Digest;
sha2::Sha256::digest(data).iter().map(|b| format!("{b:02x}")).collect()
}
fn range_server(data: Vec<u8>, n: usize) -> (u16, Arc<Mutex<Vec<Req>>>, JoinHandle<()>) {
serve(n, move |r| {
match r.header("range").and_then(|v| v.strip_prefix("bytes=")).and_then(|v| v.strip_suffix('-')).and_then(|v| v.parse::<usize>().ok()) {
Some(start) if start < data.len() => {
let cr = format!("bytes {start}-{}/{}", data.len() - 1, data.len());
response("206 Partial Content", &[("Content-Range", &cr)], &data[start..])
}
_ => response("200 OK", &[], &data),
}
})
}
#[test]
fn download_resumes_and_verifies() {
let data: Vec<u8> = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect();
let sha = sha256_hex(&data);
let dir = tempdir("dl");
let dest = dir.join("model.bin");
std::fs::write(dir.join("model.bin.part"), &data[..80_000]).unwrap();
let (port, seen, h) = range_server(data.clone(), 1);
let mut last = 0;
let mut progress = |s: u64, _t: Option<u64>| last = s;
let opts =
DownloadOptions { size: Some(data.len() as u64), sha256: Some(sha.to_uppercase()), progress: Some(&mut progress), ..Default::default() };
let got = client().download(&format!("http://127.0.0.1:{port}/m"), &dest, opts).unwrap();
h.join().unwrap();
assert_eq!(got, Downloaded { bytes: data.len() as u64, sha256: sha.clone() });
assert_eq!(last, data.len() as u64);
assert_eq!(std::fs::read(&dest).unwrap(), data);
assert!(!dir.join("model.bin.part").exists());
let seen = seen.lock().unwrap();
assert_eq!(seen[0].header("range"), Some("bytes=80000-"));
assert_eq!(seen[0].header("accept-encoding"), Some("identity"));
// a wrong hash deletes the part file and keeps no file under the real name
let dest2 = dir.join("bad.bin");
let (port, _, h) = range_server(data.clone(), 1);
let opts = DownloadOptions { sha256: Some("00".repeat(32)), ..Default::default() };
assert!(matches!(client().download(&format!("http://127.0.0.1:{port}/m"), &dest2, opts), Err(NetError::Verify(_))));
h.join().unwrap();
assert!(!dest2.exists() && !dir.join("bad.bin.part").exists());
// a cap smaller than the file
let (port, _, h) = range_server(data, 1);
let opts = DownloadOptions { max_size: 1000, ..Default::default() };
assert!(matches!(client().download(&format!("http://127.0.0.1:{port}/m"), &dir.join("big.bin"), opts), Err(NetError::TooLarge(_))));
h.join().unwrap();
let _ = std::fs::remove_dir_all(dir);
}
// --- TLS -------------------------------------------------------------------------------------
/// A test CA and a `localhost`/127.0.0.1 leaf it signed (P-256, valid until 2126), generated with
/// openssl for these tests only.
const CA_B64: &str = "MIIBjDCCATGgAwIBAgIUabPpTeBrc750uF10VHKviz4AX94wCgYIKoZIzj0EAwIwEjEQMA4GA1UEAwwHVGVzdCBDQTAgFw0yNjEwMTAwMDA1NDlaGA8yMTI2MDkxNjAwMDU0OVowEjEQMA4GA1UEAwwHVGVzdCBDQTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABB+i1UqTwg54S6Ef3QlGtBCju6tkmrJoeDPLw4AYwvfr07fhEP2asbQew8dGE8EspL4KyYwnTtsBEcocsbAah4ujYzBhMB0GA1UdDgQWBBSqjOvGOiSW/kbE/YVt+Q3Qlm7bEzAfBgNVHSMEGDAWgBSqjOvGOiSW/kbE/YVt+Q3Qlm7bEzAPBgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwICBDAKBggqhkjOPQQDAgNJADBGAiEAzHuuueM9+8Laj3rzm9LfkZuBS5cDoVwxlYPlHsfUvmYCIQDGh047dSWnC7wIfN+bryPuQ7wg023H9UdDlMBMsHDV+w==";
const LEAF_B64: &str = "MIIBqTCCAU6gAwIBAgIUHPx6OQjNVD01xDDk464Uj1xwmzcwCgYIKoZIzj0EAwIwEjEQMA4GA1UEAwwHVGVzdCBDQTAgFw0yNjEwMTAwMDA1NDlaGA8yMTI2MDkxNjAwMDU0OVowFDESMBAGA1UEAwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEFPSyEHSnzWNFWHjo6Fsccp8IUISLOIv2JcW4dwim6VlrvmcgH4fSRtMLq9JDbrs7eGp5J/7qj5VL3iqt4kGTI6N+MHwwGgYDVR0RBBMwEYIJbG9jYWxob3N0hwR/AAABMAkGA1UdEwQCMAAwEwYDVR0lBAwwCgYIKwYBBQUHAwEwHQYDVR0OBBYEFIkl10ciEZxIBMwplQXH2M/FHL6EMB8GA1UdIwQYMBaAFKqM68Y6JJb+RsT9hW35DdCWbtsTMAoGCCqGSM49BAMCA0kAMEYCIQCGcYFLw3UXrn8ea/Hfn4plLzgTQLoQzYhaA86WzbiexQIhAM/ozbpH7BFWxXjrbJfUwlFZ5A6k670RBOEwjjudfnO4";
const KEY_B64: &str = "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgP7on22FsY1CzXydImw0X41brmtJ52CZx4n9+5T6HDV+hRANCAAQU9LIQdKfNY0VYeOjoWxxynwhQhIs4i/Ylxbh3CKbpWWu+ZyAfh9JG0wur0kNuuzt4ankn/uqPlUveKq3iQZMj";
fn b64(s: &str) -> Vec<u8> {
tls::base64_decode(s).unwrap()
}
/// An HTTPS server for `n` connections answering "secure".
fn tls_serve(n: usize) -> (u16, JoinHandle<()>) {
use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer};
let config = rustls::ServerConfig::builder_with_provider(Arc::new(rustls_rustcrypto::provider()))
.with_safe_default_protocol_versions()
.unwrap()
.with_no_client_auth()
.with_single_cert(vec![CertificateDer::from(b64(LEAF_B64))], PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(b64(KEY_B64))))
.unwrap();
let config = Arc::new(config);
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
let port = listener.local_addr().unwrap().port();
let h = std::thread::spawn(move || {
for _ in 0..n {
let Ok((s, _)) = listener.accept() else { return };
s.set_read_timeout(Some(Duration::from_secs(5))).unwrap();
let conn = rustls::ServerConnection::new(config.clone()).unwrap();
let mut tls = rustls::StreamOwned::new(conn, s);
if read_req(&mut tls).is_some() {
let _ = tls.write_all(&response("200 OK", &[], b"secure"));
tls.conn.send_close_notify();
let _ = tls.flush();
}
}
});
(port, h)
}
fn get_tls(trust: Trust, port: u16) -> Result<String, NetError> {
let c = Client::new(ClientConfig { trust, stall_timeout: Duration::from_secs(5), ..ClientConfig::default() })?;
c.get(&format!("https://localhost:{port}/")).send()?.text()
}
#[test]
fn tls_trust_ca_pin_and_tofu() {
let leaf_fp = Fingerprint::of(&b64(LEAF_B64));
let (port, h) = tls_serve(5);
// untrusted: the error names the fingerprint
match get_tls(Trust::new(), port) {
Err(NetError::UntrustedCertificate { host, fingerprint }) => {
assert_eq!(host, "localhost");
assert_eq!(fingerprint, leaf_fp.to_string());
}
other => panic!("{other:?}"),
}
// the connection's own CA
assert_eq!(get_tls(Trust::new().with_ca(&b64(CA_B64)).unwrap(), port).unwrap(), "secure");
// a pinned fingerprint
assert_eq!(get_tls(Trust::new().with_pinned(leaf_fp), port).unwrap(), "secure");
// trust on first use: declined, then accepted (and pinned)
assert!(matches!(get_tls(Trust::new().with_tofu(Arc::new(|_| false)), port), Err(NetError::UntrustedCertificate { .. })));
let seen = Arc::new(Mutex::new(None));
let s2 = seen.clone();
let trust = Trust::new().with_tofu(Arc::new(move |info: &CertInfo| {
*s2.lock().unwrap() = Some(info.fingerprint);
true
}));
assert_eq!(get_tls(trust.clone(), port).unwrap(), "secure");
assert_eq!(*seen.lock().unwrap(), Some(leaf_fp));
assert_eq!(trust.pinned(), vec![leaf_fp]);
h.join().unwrap();
}
#[test]
fn ca_pem_is_accepted() {
let pem = format!("-----BEGIN CERTIFICATE-----\n{}\n-----END CERTIFICATE-----\n", CA_B64);
assert!(Trust::new().with_ca(pem.as_bytes()).is_ok());
}
// --- logs never carry secrets -----------------------------------------------------------------
struct Capture;
static LOGS: Mutex<String> = Mutex::new(String::new());
impl log::Log for Capture {
fn enabled(&self, _: &log::Metadata) -> bool {
true
}
fn log(&self, r: &log::Record) {
LOGS.lock().unwrap().push_str(&format!("{} {}\n", r.target(), r.args()));
}
fn flush(&self) {}
}
#[test]
fn secrets_never_reach_the_log_or_debug_output() {
static ONCE: std::sync::Once = std::sync::Once::new();
ONCE.call_once(|| {
let _ = log::set_logger(&Capture);
log::set_max_level(log::LevelFilter::Trace);
});
let (port, _, h) = serve(1, |_| response("401 Unauthorized", &[("Set-Cookie", "session=c00kie-secret")], b""));
let r = client()
.get(&format!("http://127.0.0.1:{port}/api/x?key=share-secret"))
.bearer("bearer-secret")
.header("x-api-key", "apikey-secret")
.send()
.unwrap();
let debug = format!("{r:?}");
let err = r.error_for_status().unwrap_err().to_string();
h.join().unwrap();
let logs = LOGS.lock().unwrap().clone();
assert!(logs.contains("net: GET"), "logging works: {logs}");
for text in [&logs, &debug, &err] {
for secret in ["bearer-secret", "apikey-secret", "share-secret", "c00kie-secret"] {
assert!(!text.contains(secret), "{secret} leaked into {text}");
}
}
}
crates/net/src/tls.rs
+281
View File
@@ -0,0 +1,281 @@
//! Which server certificates a connection trusts.
//!
//! A certificate passes when it chains to the Mozilla roots (`webpki-roots`) or to one of the
//! connection's extra CA certificates, or when the SHA-256 fingerprint of the server's own
//! certificate is pinned. Otherwise, when a trust-on-first-use hook is set, the hook sees the
//! certificate (host, fingerprint, why it failed) and decides; an accepted fingerprint is pinned
//! for the rest of the [`Trust`]'s life and is readable through [`Trust::pinned`] so the caller
//! can store it with the account. Home servers with self-signed certificates (common for Immich)
//! work this way without ever turning verification off: handshake signatures are always checked.
use std::sync::{Arc, Mutex, PoisonError};
use rustls::client::WebPkiServerVerifier;
use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};
use rustls::crypto::CryptoProvider;
use rustls::pki_types::{CertificateDer, ServerName, UnixTime};
use rustls::{DigitallySignedStruct, RootCertStore, SignatureScheme};
use sha2::{Digest, Sha256};
use crate::NetError;
/// SHA-256 of a DER certificate.
#[derive(Clone, Copy, PartialEq, Eq, Hash)]
pub struct Fingerprint(pub [u8; 32]);
impl Fingerprint {
pub fn of(der: &[u8]) -> Fingerprint {
Fingerprint(Sha256::digest(der).into())
}
/// Parse the `AB:CD:…` form (colons and case optional).
pub fn parse(s: &str) -> Option<Fingerprint> {
let hex: String = s.chars().filter(|c| *c != ':').collect();
if hex.len() != 64 {
return None;
}
let mut out = [0u8; 32];
for (i, b) in out.iter_mut().enumerate() {
*b = u8::from_str_radix(hex.get(i * 2..i * 2 + 2)?, 16).ok()?;
}
Some(Fingerprint(out))
}
}
impl std::fmt::Display for Fingerprint {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
for (i, b) in self.0.iter().enumerate() {
if i > 0 {
f.write_str(":")?;
}
write!(f, "{b:02X}")?;
}
Ok(())
}
}
impl std::fmt::Debug for Fingerprint {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Fingerprint({self})")
}
}
/// What the trust-on-first-use hook is shown.
#[derive(Debug, Clone)]
pub struct CertInfo {
pub host: String,
pub fingerprint: Fingerprint,
/// Why normal verification refused it (self-signed, unknown issuer, expired, wrong name…).
pub reason: String,
/// The server's certificate (DER), for a details view.
pub der: Vec<u8>,
}
/// Decides about a certificate nothing else vouches for: `true` accepts (and pins) it.
/// Called on the requesting thread during the handshake; a UI typically answers from a
/// fingerprint the user confirmed earlier, or fails the request and asks.
pub type TofuHook = Arc<dyn Fn(&CertInfo) -> bool + Send + Sync>;
/// The trust settings of one connection (account). Cheap to clone; clones share the pins.
#[derive(Clone, Default)]
pub struct Trust {
extra_roots: Vec<Vec<u8>>,
pinned: Arc<Mutex<Vec<Fingerprint>>>,
tofu: Option<TofuHook>,
}
impl std::fmt::Debug for Trust {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Trust")
.field("extra_roots", &self.extra_roots.len())
.field("pinned", &self.pinned())
.field("tofu", &self.tofu.is_some())
.finish()
}
}
impl Trust {
/// Mozilla roots only.
pub fn new() -> Trust {
Trust::default()
}
/// Also trust certificates issued by this CA (DER or PEM).
pub fn with_ca(mut self, cert: &[u8]) -> Result<Trust, NetError> {
let ders = if cert.starts_with(b"-----") { pem_certificates(cert)? } else { vec![cert.to_vec()] };
if ders.is_empty() {
return Err(NetError::Tls("no certificate in the CA file".into()));
}
self.extra_roots.extend(ders);
Ok(self)
}
/// Trust the server certificate with this fingerprint (one the user confirmed earlier).
pub fn with_pinned(self, fp: Fingerprint) -> Trust {
self.pin(fp);
self
}
/// Ask `hook` about certificates nothing else vouches for.
pub fn with_tofu(mut self, hook: TofuHook) -> Trust {
self.tofu = Some(hook);
self
}
/// The pinned fingerprints, including those accepted by the hook since.
pub fn pinned(&self) -> Vec<Fingerprint> {
self.pinned.lock().unwrap_or_else(PoisonError::into_inner).clone()
}
fn pin(&self, fp: Fingerprint) {
let mut p = self.pinned.lock().unwrap_or_else(PoisonError::into_inner);
if !p.contains(&fp) {
p.push(fp);
}
}
fn is_pinned(&self, fp: &Fingerprint) -> bool {
self.pinned.lock().unwrap_or_else(PoisonError::into_inner).contains(fp)
}
/// The rustls client configuration for this trust.
pub(crate) fn client_config(&self) -> Result<Arc<rustls::ClientConfig>, NetError> {
let provider = Arc::new(rustls_rustcrypto::provider());
let mut roots = RootCertStore { roots: webpki_roots::TLS_SERVER_ROOTS.to_vec() };
for der in &self.extra_roots {
roots.add(CertificateDer::from(der.clone())).map_err(|e| NetError::Tls(format!("CA certificate: {e}")))?;
}
let webpki =
WebPkiServerVerifier::builder_with_provider(Arc::new(roots), provider.clone()).build().map_err(|e| NetError::Tls(e.to_string()))?;
let verifier = Arc::new(Verifier { webpki, provider: provider.clone(), trust: self.clone() });
let config = rustls::ClientConfig::builder_with_provider(provider)
.with_safe_default_protocol_versions()
.map_err(|e| NetError::Tls(e.to_string()))?
.dangerous()
.with_custom_certificate_verifier(verifier)
.with_no_client_auth();
Ok(Arc::new(config))
}
}
/// Pull the CERTIFICATE blocks out of a PEM file.
fn pem_certificates(pem: &[u8]) -> Result<Vec<Vec<u8>>, NetError> {
let text = std::str::from_utf8(pem).map_err(|_| NetError::Tls("CA file is not text".into()))?;
let mut out = Vec::new();
let mut rest = text;
while let Some(start) = rest.find("-----BEGIN CERTIFICATE-----") {
let body = rest.get(start + 27..).unwrap_or_default();
let end = body.find("-----END CERTIFICATE-----").ok_or_else(|| NetError::Tls("unterminated PEM certificate".into()))?;
let b64: String = body.get(..end).unwrap_or_default().chars().filter(|c| !c.is_whitespace()).collect();
out.push(base64_decode(&b64).ok_or_else(|| NetError::Tls("bad base64 in PEM certificate".into()))?);
rest = body.get(end..).unwrap_or_default();
}
Ok(out)
}
pub(crate) fn base64_decode(s: &str) -> Option<Vec<u8>> {
let mut out = Vec::with_capacity(s.len() / 4 * 3);
let (mut acc, mut bits) = (0u32, 0u32);
for c in s.bytes() {
let v = match c {
b'A'..=b'Z' => c - b'A',
b'a'..=b'z' => c - b'a' + 26,
b'0'..=b'9' => c - b'0' + 52,
b'+' => 62,
b'/' => 63,
b'=' => break,
_ => return None,
};
acc = (acc << 6) | u32::from(v);
bits += 6;
if bits >= 8 {
bits -= 8;
out.push((acc >> bits) as u8);
acc &= (1 << bits) - 1;
}
}
Some(out)
}
#[derive(Debug)]
struct Verifier {
webpki: Arc<WebPkiServerVerifier>,
provider: Arc<CryptoProvider>,
trust: Trust,
}
impl ServerCertVerifier for Verifier {
fn verify_server_cert(
&self,
end_entity: &CertificateDer<'_>,
intermediates: &[CertificateDer<'_>],
server_name: &ServerName<'_>,
ocsp: &[u8],
now: UnixTime,
) -> Result<ServerCertVerified, rustls::Error> {
let reason = match self.webpki.verify_server_cert(end_entity, intermediates, server_name, ocsp, now) {
Ok(ok) => return Ok(ok),
Err(e) => e.to_string(),
};
let fp = Fingerprint::of(end_entity.as_ref());
if self.trust.is_pinned(&fp) {
return Ok(ServerCertVerified::assertion());
}
let host = server_name.to_str().into_owned();
if let Some(hook) = &self.trust.tofu {
let info = CertInfo { host: host.clone(), fingerprint: fp, reason, der: end_entity.as_ref().to_vec() };
if hook(&info) {
log::info!("net: accepted certificate {fp} for {host}");
self.trust.pin(fp);
return Ok(ServerCertVerified::assertion());
}
}
Err(rustls::Error::Other(rustls::OtherError(Arc::new(NetError::UntrustedCertificate { host, fingerprint: fp.to_string() }))))
}
fn verify_tls12_signature(
&self,
message: &[u8],
cert: &CertificateDer<'_>,
dss: &DigitallySignedStruct,
) -> Result<HandshakeSignatureValid, rustls::Error> {
rustls::crypto::verify_tls12_signature(message, cert, dss, &self.provider.signature_verification_algorithms)
}
fn verify_tls13_signature(
&self,
message: &[u8],
cert: &CertificateDer<'_>,
dss: &DigitallySignedStruct,
) -> Result<HandshakeSignatureValid, rustls::Error> {
rustls::crypto::verify_tls13_signature(message, cert, dss, &self.provider.signature_verification_algorithms)
}
fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
self.provider.signature_verification_algorithms.supported_schemes()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn fingerprint_round_trip() {
let fp = Fingerprint::of(b"abc");
let s = fp.to_string();
assert_eq!(s.len(), 95);
assert_eq!(Fingerprint::parse(&s), Some(fp));
assert_eq!(Fingerprint::parse(&s.replace(':', "").to_lowercase()), Some(fp));
assert_eq!(Fingerprint::parse("zz"), None);
assert_eq!(Fingerprint::parse(&"é".repeat(32)), None);
}
#[test]
fn base64() {
assert_eq!(base64_decode("aGVsbG8=").unwrap(), b"hello");
assert_eq!(base64_decode("!!"), None);
assert!(Trust::new().with_ca(b"-----BEGIN CERTIFICATE-----\nAAA").is_err());
assert!(Trust::new().with_ca(b"-----nothing").is_err());
}
}
crates/net/src/url.rs
+154
View File
@@ -0,0 +1,154 @@
//! `http://` and `https://` URLs: just enough parsing to connect, send a request line and follow
//! redirects. Hostile input never panics; anything odd is a [`NetError::BadUrl`].
use crate::NetError;
/// Longest URL accepted.
const MAX_URL: usize = 16 * 1024;
/// A parsed `http://` or `https://` URL.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Url {
/// `https://`.
pub tls: bool,
/// Host name or IP address (an IPv6 address without brackets).
pub host: String,
pub port: u16,
/// Path and query, starting with `/`.
pub path: String,
}
impl Url {
pub fn parse(s: &str) -> Result<Url, NetError> {
let bad = |why: &str| NetError::BadUrl(why.to_string());
if s.len() > MAX_URL {
return Err(bad("longer than 16 KiB"));
}
let s = s.trim();
if s.chars().any(|c| c.is_control() || c == ' ') {
return Err(bad("contains spaces or control characters"));
}
let lower = s.get(..8).unwrap_or(s).to_ascii_lowercase();
let (tls, rest) = if lower.starts_with("https://") {
(true, s.get(8..).unwrap_or_default())
} else if lower.starts_with("http://") {
(false, s.get(7..).unwrap_or_default())
} else {
return Err(bad("only http:// and https:// are supported"));
};
let rest = rest.split('#').next().unwrap_or_default();
let (authority, path) = match rest.find(['/', '?']) {
Some(i) => (rest.get(..i).unwrap_or_default(), rest.get(i..).unwrap_or_default()),
None => (rest, ""),
};
if authority.contains('@') {
return Err(bad("credentials in URLs are not supported"));
}
let default_port = if tls { 443 } else { 80 };
let (host, port) = if let Some(v6) = authority.strip_prefix('[') {
let end = v6.find(']').ok_or_else(|| bad("unterminated IPv6 address"))?;
let host = v6.get(..end).unwrap_or_default();
let after = v6.get(end + 1..).unwrap_or_default();
let port = match after.strip_prefix(':') {
Some(p) => parse_port(p)?,
None if after.is_empty() => default_port,
None => return Err(bad("text after the IPv6 address")),
};
(host.to_string(), port)
} else {
match authority.rsplit_once(':') {
Some((h, p)) => (h.to_string(), parse_port(p)?),
None => (authority.to_string(), default_port),
}
};
if host.is_empty() {
return Err(bad("no host"));
}
if !host.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_' | ':')) {
return Err(bad("host has unsupported characters (use the punycode form)"));
}
let path = if path.starts_with('?') {
format!("/{path}")
} else if path.is_empty() {
"/".to_string()
} else {
path.to_string()
};
Ok(Url { tls, host: host.to_ascii_lowercase(), port, path })
}
/// The `Host` header value (port only when not the default).
pub fn host_header(&self) -> String {
let host = if self.host.contains(':') { format!("[{}]", self.host) } else { self.host.clone() };
if self.port == self.default_port() { host } else { format!("{host}:{}", self.port) }
}
fn default_port(&self) -> u16 {
if self.tls { 443 } else { 80 }
}
/// `scheme://host[:port]`, to compare origins (credentials never cross origins).
pub fn origin(&self) -> String {
format!("{}://{}", if self.tls { "https" } else { "http" }, self.host_header())
}
/// Resolve a `Location` header against this URL.
pub fn join(&self, location: &str) -> Result<Url, NetError> {
let l = location.trim();
let lower = l.get(..8).unwrap_or(l).to_ascii_lowercase();
if lower.starts_with("http://") || lower.starts_with("https://") {
return Url::parse(l);
}
if let Some(rest) = l.strip_prefix("//") {
return Url::parse(&format!("{}://{rest}", if self.tls { "https" } else { "http" }));
}
let path = if l.starts_with('/') {
l.to_string()
} else {
let base = self.path.split('?').next().unwrap_or("/");
let dir = base.rsplit_once('/').map(|(d, _)| d).unwrap_or("");
format!("{dir}/{l}")
};
Url::parse(&format!("{}{path}", self.origin()))
}
}
impl std::fmt::Display for Url {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}{}", self.origin(), self.path)
}
}
fn parse_port(p: &str) -> Result<u16, NetError> {
match p.parse::<u16>() {
Ok(n) if n > 0 => Ok(n),
_ => Err(NetError::BadUrl(format!("bad port {p:?}"))),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_and_joins() {
let u = Url::parse("HTTPS://Photos.Example:8443/api/x?y=1#frag").unwrap();
assert_eq!((u.tls, u.host.as_str(), u.port, u.path.as_str()), (true, "photos.example", 8443, "/api/x?y=1"));
assert_eq!(u.host_header(), "photos.example:8443");
assert_eq!(u.join("/b").unwrap().to_string(), "https://photos.example:8443/b");
assert_eq!(u.join("c?d").unwrap().path, "/api/c?d");
assert_eq!(u.join("//other/z").unwrap().to_string(), "https://other/z");
let v6 = Url::parse("http://[::1]:2283").unwrap();
assert_eq!((v6.host.as_str(), v6.port, v6.path.as_str()), ("::1", 2283, "/"));
assert_eq!(v6.host_header(), "[::1]:2283");
assert_eq!(Url::parse("http://h?q").unwrap().path, "/?q");
}
#[test]
fn hostile_urls_are_errors() {
for s in ["ftp://x", "http://", "http://a b", "http://u:p@h/", "http://h:0", "http://h:99999", "http://[::1", "http://h\r\n/", "http://é/"] {
assert!(Url::parse(s).is_err(), "{s}");
}
assert!(Url::parse(&"h".repeat(20_000)).is_err());
}
}
xtask/src/layers.rs
+5
View File
@@ -40,6 +40,9 @@ pub const TABLE: &[(&str, Class)] = &[
("tiff", Class::Layer(0)),
("sysmem", Class::Layer(0)),
("fetch", Class::Layer(0)),
("net", Class::Layer(0)),
("credentials", Class::Layer(0)),
("hash", Class::Layer(0)),
("denoise-core", Class::Layer(1)),
("denoise", Class::Layer(1)),
("heif", Class::Layer(0)),
@@ -82,6 +85,8 @@ pub const INTRA_LAYER_ORDER: &[&[&str]] = &[
&["brand", "tiff"],
&["brand", "sysmem"],
&["brand", "fetch"],
&["brand", "net"],
&["brand", "credentials"],
&["brand", "heif"],
&["geom", "color", "raster"],
&["tiff", "raster"],