//! Pinned PngSuite files: verify bounded HTTPS downloads before publishing, without extracting archives. use std::collections::HashSet; use std::fs::{self, File, OpenOptions}; use std::io::Read; use std::path::{Path, PathBuf}; use std::process::Command; use crate::{root, run, sha256}; const SOURCE: &str = "https://raw.githubusercontent.com/lunapaint/pngsuite/8cd768dd0d0063195174d0d01cacbd5a7d1e5605"; const MANIFEST: &str = include_str!("../pngsuite.sha256"); const MAX_FILE: u64 = 1024 * 1024; const MAX_TOTAL: u64 = 16 * MAX_FILE; const MAX_FILES: usize = 256; #[derive(Debug)] struct Entry<'a> { sha: &'a str, bytes: u64, name: &'a str, } fn entries(manifest: &str) -> Result>, String> { let mut result = Vec::new(); let mut names = HashSet::new(); let mut total = 0u64; for line in manifest.lines().filter(|l| !l.trim().is_empty() && !l.starts_with('#')) { let fields: Vec<_> = line.split_whitespace().collect(); let [sha, size, name] = fields.as_slice() else { return Err("invalid PngSuite manifest row".into()) }; let png = name.strip_suffix(".png").is_some_and(|s| !s.is_empty() && s.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit())); if !png && !matches!(*name, "LICENSE" | "PngSuite.README") { return Err(format!("unexpected PngSuite filename: {name}")); } if sha.len() != 64 || !sha.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) { return Err(format!("invalid PngSuite sha256: {name}")); } let bytes = size.parse::().map_err(|_| format!("invalid PngSuite size: {name}"))?; if bytes == 0 || bytes > MAX_FILE || bytes > MAX_TOTAL.saturating_sub(total) || result.len() >= MAX_FILES || !names.insert(*name) { return Err(format!("PngSuite manifest budget or duplicate filename: {name}")); } total += bytes; result.push(Entry { sha, bytes, name }); } if !result.iter().any(|e| e.name.ends_with(".png")) { return Err("PngSuite manifest has no PNG files".into()); } Ok(result) } fn verified(path: &Path, entry: &Entry<'_>) -> Result { let metadata = match fs::symlink_metadata(path) { Ok(m) => m, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(false), Err(e) => return Err(format!("{}: {e}", path.display())), }; if !metadata.file_type().is_file() { return Err(format!("{}: expected a regular corpus file", path.display())); } if metadata.len() != entry.bytes { return Ok(false); } let mut data = Vec::new(); File::open(path).and_then(|f| f.take(entry.bytes + 1).read_to_end(&mut data)).map_err(|e| format!("{}: {e}", path.display()))?; Ok(data.len() as u64 == entry.bytes && sha256::hex(&data) == entry.sha) } struct Download(PathBuf); impl Drop for Download { fn drop(&mut self) { // Best-effort removal of our disposable download on success, rejection or network failure. let _ = fs::remove_file(&self.0); } } fn fetch_into(dest: &Path, manifest: &str, mut download: impl FnMut(&str, &Path, u64) -> Result<(), String>) -> Result { let entries = entries(manifest)?; fs::create_dir_all(dest).map_err(|e| format!("{}: {e}", dest.display()))?; if !fs::symlink_metadata(dest).map_err(|e| e.to_string())?.file_type().is_dir() { return Err("PngSuite destination must be a regular directory".into()); } let mut count = 0; for entry in entries { let out = dest.join(entry.name); if !verified(&out, &entry)? { let temporary = dest.join(format!(".{}-{}.download", entry.name, std::process::id())); // Reserve a new regular file, never reuse somebody else's stale download or link. OpenOptions::new().write(true).create_new(true).open(&temporary).map_err(|e| format!("{}: {e}", temporary.display()))?; let temporary = Download(temporary); let folder = if entry.name.ends_with(".png") { "/png" } else { "" }; download(&format!("{SOURCE}{folder}/{}", entry.name), &temporary.0, entry.bytes)?; if !verified(&temporary.0, &entry)? { return Err(format!("{}: size or sha256 does not match xtask/pngsuite.sha256", entry.name)); } fs::rename(&temporary.0, &out).map_err(|e| format!("publish {}: {e}", out.display()))?; } count += usize::from(entry.name.ends_with(".png")); } Ok(count) } fn supported_curl(version: &str) -> bool { let Some(version) = version.strip_prefix("curl ").and_then(|s| s.split_whitespace().next()) else { return false }; let mut parts = version.split('.'); let major = parts.next().and_then(|s| s.parse::().ok()); let minor = parts.next().and_then(|s| s.parse::().ok()); matches!((major, minor), (Some(major), Some(minor)) if (major, minor) >= (8, 4)) } pub fn fetch() -> Result<(), String> { let dest = root().join("corpus/pngsuite"); let mut checked_curl = false; let count = fetch_into(&dest, MANIFEST, |url, path, bytes| { if !checked_curl { // Older curl only bounds responses whose length is known before downloading. let version = Command::new("curl").arg("--version").output().map_err(|e| format!("curl --version: {e}"))?; if !version.status.success() || !supported_curl(&String::from_utf8_lossy(&version.stdout)) { return Err("PngSuite downloading requires curl 8.4 or newer to bound unknown-length responses".into()); } checked_curl = true; } let mut curl = Command::new("curl"); curl.args([ "-fsSL", "--proto", "=https", "--proto-redir", "=https", "--connect-timeout", "10", "--max-time", "60", "--retry", "2", "--max-filesize", ]) .arg(bytes.to_string()) .arg("-o") .arg(path) .arg(url); run(curl, &format!("curl {url}")) })?; println!("PngSuite: {count} PNG files verified in {}", dest.display()); Ok(()) } #[cfg(test)] mod tests { use super::*; use std::sync::atomic::{AtomicUsize, Ordering}; static NEXT: AtomicUsize = AtomicUsize::new(0); struct Temp(PathBuf); impl Temp { fn new() -> Self { let path = std::env::temp_dir().join(format!("vectorcraft-pngsuite-{}-{}", std::process::id(), NEXT.fetch_add(1, Ordering::Relaxed))); fs::create_dir(&path).unwrap(); Self(path) } } impl Drop for Temp { fn drop(&mut self) { let _ = fs::remove_dir_all(&self.0); } } fn manifest(data: &[u8]) -> String { format!("{} {} test.png\n", sha256::hex(data), data.len()) } #[test] fn substituted_and_oversized_downloads_never_replace_corpus_files() { let temp = Temp::new(); let out = temp.0.join("test.png"); let good = b"original PNG oracle"; for replacement in [b"substitutePNGoracle".as_slice(), &[0; 100]] { fs::write(&out, b"old corpus data").unwrap(); let error = fetch_into(&temp.0, &manifest(good), |_, path, limit| { assert_eq!(limit, good.len() as u64); fs::write(path, replacement).unwrap(); Ok(()) }) .unwrap_err(); assert!(error.contains("size or sha256")); assert_eq!(fs::read(&out).unwrap(), b"old corpus data"); assert_eq!(fs::read_dir(&temp.0).unwrap().count(), 1); } } #[test] fn verified_cache_is_reused_and_corrupt_cache_is_repaired() { let temp = Temp::new(); let data = b"verified oracle"; let m = manifest(data); let mut requests = 0; for _ in 0..2 { assert_eq!( fetch_into(&temp.0, &m, |url, path, _| { requests += 1; assert!(url.starts_with(SOURCE) && url.ends_with("/png/test.png")); fs::write(path, data).unwrap(); Ok(()) }) .unwrap(), 1 ); } assert_eq!(requests, 1); fs::write(temp.0.join("test.png"), b"wrong oracle!!!").unwrap(); fetch_into(&temp.0, &m, |_, path, _| { fs::write(path, data).unwrap(); Ok(()) }) .unwrap(); assert_eq!(fs::read(temp.0.join("test.png")).unwrap(), data); } #[test] fn network_failures_clean_staging_and_nonregular_targets_are_rejected() { let temp = Temp::new(); let m = manifest(b"png"); assert!( fetch_into(&temp.0, &m, |_, path, _| { fs::write(path, b"partial").unwrap(); Err("network failed".into()) }) .is_err() ); assert_eq!(fs::read_dir(&temp.0).unwrap().count(), 0); fs::create_dir(temp.0.join("test.png")).unwrap(); let e = fetch_into(&temp.0, &m, |_, _, _| panic!("must not download over a nonregular file")).unwrap_err(); assert!(e.contains("regular corpus file")); } #[test] fn stale_download_is_never_overwritten() { let temp = Temp::new(); let stale = temp.0.join(format!(".test.png-{}.download", std::process::id())); fs::write(&stale, b"keep this earlier download").unwrap(); assert!(fetch_into(&temp.0, &manifest(b"png"), |_, _, _| panic!("must not reuse an existing staging file")).is_err()); assert_eq!(fs::read(stale).unwrap(), b"keep this earlier download"); assert!(!temp.0.join("test.png").exists()); } #[test] fn manifest_rejects_unexpected_names_duplicates_and_resource_budgets() { let sha = sha256::hex(b"png"); for name in ["../test.png", "a/b.png", "a\\b.png", "a:stream.png", "script.exe", "a..png"] { assert!(entries(&format!("{sha} 3 {name}\n")).is_err()); } for m in [format!("{sha} 0 a.png"), format!("{sha} {} a.png", MAX_FILE + 1), format!("{sha} 3 a.png\n{sha} 3 a.png"), "bad 3 a.png".into()] { assert!(entries(&m).is_err()); } let m: String = (0..17).map(|i| format!("{sha} {MAX_FILE} a{i}.png\n")).collect(); assert!(entries(&m).is_err()); let m: String = (0..257).map(|i| format!("{sha} 1 a{i}.png\n")).collect(); assert!(entries(&m).is_err()); } #[test] fn pinned_manifest_contains_the_original_175_pngs_and_redistribution_notice() { let e = entries(MANIFEST).unwrap(); assert_eq!(e.iter().filter(|e| e.name.ends_with(".png")).count(), 175); assert!(e.iter().any(|e| e.name == "LICENSE")); assert!(e.iter().any(|e| e.name == "PngSuite.README")); } #[test] fn old_curl_without_unknown_length_download_limits_is_rejected() { for version in ["curl 7.88.1", "curl 8.3.0", "curl invalid", "not curl"] { assert!(!supported_curl(version)); } for version in ["curl 8.4.0 libcurl/8.4.0", "curl 8.21.0 (Windows)", "curl 9.0.0"] { assert!(supported_curl(version)); } } }