//! Small, read-only SQLite reader used by Lightroom catalog import. //! //! This is a clean-room implementation based on SQLite's public file-format //! description: (sections 1.6, 1.7, //! 2.1, 2.2, 2.3, 2.4 and 2.5). It is deliberately a page reader, rather //! than a SQL engine. It never writes its input and has no SQLite runtime //! dependency. use std::collections::{HashMap, HashSet}; const MAX_PAGE_ID: u32 = 16_777_216; const MAX_COLUMNS: usize = 16_384; const MAX_ROWS: usize = 1_000_000; const MAX_CELLS: usize = 4_000_000; const MAX_PAYLOAD: usize = 64 * 1024 * 1024; const MAX_TABLE_PAYLOAD: usize = 256 * 1024 * 1024; const MAX_OVERFLOW_PAGES: usize = 1_048_576; const MAX_BTREE_DEPTH: usize = 64; /// Values supported by SQLite table records. #[derive(Clone, Debug, PartialEq)] pub enum Value { Null, Integer(i64), Real(f64), Text(String), Blob(Vec), } /// A row returned by [`Database::read_table`]. #[derive(Clone, Debug, PartialEq)] pub struct Record { pub values: Vec, } /// A table discovered in `sqlite_schema`. #[derive(Clone, Debug, PartialEq, Eq)] pub struct LiveTable { pub name: String, pub rootpage: u32, pub column_names: Option>, } /// Read-only SQLite database image, with a committed WAL overlay if supplied. pub struct Database { main: Vec, overlay: HashMap>, page_size: usize, reserved: usize, encoding: TextEncoding, page_count: u32, tables: Vec, rowid_aliases: HashMap, } #[derive(Clone, Copy)] enum TextEncoding { Utf8, Utf16Le, Utf16Be, } struct WalkState { seen: HashSet, out: Vec, payload_total: usize, } struct WalOverlay { page_size: usize, frame_count: usize, pages: HashMap>, page_count: Option, } struct SchemaInfo { tables: Vec, aliases: HashMap, } impl Database { /// Open a main database image and an already checksum-validated WAL prefix. /// /// The WAL argument must include its 32-byte header and only complete /// frames through a commit. Frames are overlaid by page number, with the /// final commit's database-size field defining the visible page count. pub fn open_with_wal(main: Vec, committed_wal: &[u8]) -> Result { let main_header = main.get(..100).ok_or("truncated SQLite header")?; if main_header.get(..16) != Some(b"SQLite format 3\0") { return Err("invalid SQLite header signature".into()); } let page_size = sqlite_page_size(be_u16(main_header, 16)?)?; validate_header_fields(main_header, page_size)?; if main.len() < page_size || !main.len().is_multiple_of(page_size) { return Err("SQLite image is not a whole number of pages".into()); } let main_pages = u32::try_from(main.len() / page_size).map_err(|_| "SQLite image is too large")?; if main_pages == 0 || main_pages > MAX_PAGE_ID { return Err("invalid SQLite image page count".into()); } let mut overlay = HashMap::new(); let mut wal_page_count = None; if !committed_wal.is_empty() { let parsed = parse_wal(committed_wal, page_size)?; if parsed.page_size != page_size { return Err("WAL page size mismatch".into()); } if parsed.frame_count > MAX_CELLS { return Err("WAL contains too many frames".into()); } overlay = parsed.pages; wal_page_count = parsed.page_count; } let effective_header = overlay.get(&1).map(Vec::as_slice).unwrap_or(main_header); if effective_header.get(..16) != Some(b"SQLite format 3\0") { return Err("invalid SQLite header page in WAL".into()); } let effective_size = sqlite_page_size(be_u16(effective_header, 16)?)?; if effective_size != page_size { return Err("WAL page 1 changes SQLite page size".into()); } let reserved = validate_header_fields(effective_header, page_size)?; let header_pages = be_u32(effective_header, 28)?; let header_valid = be_u32(effective_header, 24)? != 0 && be_u32(effective_header, 24)? == be_u32(effective_header, 92)?; if header_valid && header_pages > main_pages && wal_page_count.is_none() { return Err("SQLite header page count exceeds image".into()); } let mut page_count = if header_valid && header_pages != 0 { header_pages } else { main_pages }; if let Some(count) = wal_page_count { page_count = count; } if page_count == 0 || page_count > MAX_PAGE_ID { return Err("invalid visible SQLite page count".into()); } let encoding = match be_u32(effective_header, 56)? { 0 | 1 => TextEncoding::Utf8, 2 => TextEncoding::Utf16Le, 3 => TextEncoding::Utf16Be, _ => return Err("unsupported SQLite text encoding".into()), }; let mut db = Self { main, overlay, page_size, reserved, encoding, page_count, tables: Vec::new(), rowid_aliases: HashMap::new() }; let schema = db.parse_schema()?; db.tables = schema.tables; db.rowid_aliases = schema.aliases; Ok(db) } /// Return tables from `sqlite_schema` in schema order. pub fn live_tables(&self) -> Result, String> { Ok(self.tables.clone()) } /// Read a table B-tree, decoding up to `ncols` values per record. pub fn read_table(&self, rootpage: u32, ncols: usize) -> Result, String> { if !(1..=MAX_PAGE_ID).contains(&rootpage) || rootpage > self.page_count { return Err("table root page is outside database".into()); } if ncols > MAX_COLUMNS { return Err("table has too many columns".into()); } let alias = self.rowid_aliases.get(&rootpage).copied(); let mut state = WalkState { seen: HashSet::new(), out: Vec::new(), payload_total: 0 }; self.visit_table(rootpage, ncols, alias, 0, &mut state)?; Ok(state.out) } fn page(&self, page_id: u32) -> Result<&[u8], String> { if page_id == 0 || page_id > self.page_count { return Err(format!("page {page_id} is outside database")); } if let Some(page) = self.overlay.get(&page_id) { return Ok(page); } let id = usize::try_from(page_id).map_err(|_| "page number does not fit host")?; let start = id.checked_sub(1).and_then(|n| n.checked_mul(self.page_size)).ok_or("page offset overflow")?; let end = start.checked_add(self.page_size).ok_or("page offset overflow")?; self.main.get(start..end).ok_or_else(|| format!("page {page_id} is truncated")) } fn visit_table(&self, page_id: u32, ncols: usize, alias: Option, depth: usize, state: &mut WalkState) -> Result<(), String> { if depth > MAX_BTREE_DEPTH { return Err("SQLite B-tree is too deep".into()); } if !state.seen.insert(page_id) { return Err("SQLite B-tree contains a cycle or shared page".into()); } let page = self.page(page_id)?; let usable = self.page_size.checked_sub(self.reserved).ok_or("invalid usable page size")?; let base = if page_id == 1 { 100 } else { 0 }; let content = page.get(..usable).ok_or("truncated usable SQLite page")?; let kind = *content.get(base).ok_or("truncated B-tree header")?; let cells = usize::from(be_u16_at(content, base + 3)?); if cells > MAX_CELLS || state.out.len().saturating_add(cells) > MAX_ROWS.saturating_add(MAX_CELLS) { return Err("SQLite B-tree contains too many cells".into()); } match kind { 0x0d => { let ptr_start = base.checked_add(8).ok_or("B-tree header overflow")?; let ptr_bytes = cells.checked_mul(2).ok_or("B-tree pointer-array overflow")?; let ptr_end = ptr_start.checked_add(ptr_bytes).ok_or("B-tree pointer-array overflow")?; if ptr_end > usable { return Err("truncated B-tree pointer array".into()); } for i in 0..cells { let p = ptr_start.checked_add(i.checked_mul(2).ok_or("cell offset overflow")?).ok_or("cell offset overflow")?; let cell = usize::from(be_u16_at(content, p)?); if cell < ptr_end || cell >= usable { return Err("invalid B-tree cell pointer".into()); } let mut at = cell; let payload_len = usize_from_u64(read_varint(content, &mut at)?, "payload length")?; if payload_len > MAX_PAYLOAD { return Err("SQLite record payload is too large".into()); } let rowid = i64::from_be_bytes(read_varint(content, &mut at)?.to_be_bytes()); state.payload_total = state.payload_total.checked_add(payload_len).ok_or("SQLite table payload total overflow")?; if state.payload_total > MAX_TABLE_PAYLOAD { return Err("SQLite table payload exceeds bounded import size".into()); } let payload = self.payload(page, at, payload_len)?; let mut values = decode_record(&payload, ncols, self.encoding)?; if let Some(index) = alias { if index >= values.len() { return Err("INTEGER PRIMARY KEY column is missing from record".into()); } if let Some(value @ Value::Null) = values.get_mut(index) { *value = Value::Integer(rowid); } } if state.out.len() >= MAX_ROWS { return Err("SQLite table contains more than one million records".into()); } state.out.push(Record { values }); } Ok(()) } 0x05 => { let ptr_start = base.checked_add(12).ok_or("B-tree header overflow")?; let ptr_bytes = cells.checked_mul(2).ok_or("B-tree pointer-array overflow")?; let ptr_end = ptr_start.checked_add(ptr_bytes).ok_or("B-tree pointer-array overflow")?; if ptr_end > usable { return Err("truncated interior B-tree pointer array".into()); } for i in 0..cells { let p = ptr_start.checked_add(i.checked_mul(2).ok_or("cell offset overflow")?).ok_or("cell offset overflow")?; let cell = usize::from(be_u16_at(content, p)?); if cell < ptr_end || cell >= usable { return Err("invalid interior B-tree cell pointer".into()); } let child = be_u32_at(content, cell)?; if child == 0 { return Err("interior B-tree cell has null child".into()); } let mut key_at = cell.checked_add(4).ok_or("interior cell overflow")?; let _ = read_varint(content, &mut key_at)?; self.visit_table(child, ncols, alias, depth + 1, state)?; } let right = be_u32_at(content, base + 8)?; if right == 0 { return Err("interior B-tree has null right child".into()); } self.visit_table(right, ncols, alias, depth + 1, state) } 0x02 | 0x0a => Err("SQLite index B-trees are unsupported".into()), other => Err(format!("unsupported SQLite B-tree page type 0x{other:02x}")), } } fn payload(&self, page: &[u8], cell_at: usize, payload_len: usize) -> Result, String> { let usable = self.page_size.checked_sub(self.reserved).ok_or("invalid usable page size")?; if usable < 8 { return Err("SQLite usable page size is too small".into()); } let max_local = usable.checked_sub(35).ok_or("SQLite usable page size is too small")?; let min_local = usable .checked_sub(12) .ok_or("SQLite usable page size is too small")? .saturating_mul(32) .checked_div(255) .ok_or("SQLite local payload calculation failed")? .saturating_sub(23); let local = if payload_len <= max_local { payload_len } else { let span = usable.checked_sub(4).ok_or("SQLite usable page size is too small")?; let candidate = min_local.saturating_add(payload_len.checked_sub(min_local).ok_or("SQLite payload length is too small")? % span); if candidate > max_local { min_local } else { candidate } }; let local_end = cell_at.checked_add(local).ok_or("SQLite cell payload overflow")?; if local_end > usable { return Err("truncated SQLite cell payload".into()); } let overflow_bytes = self .page_count .saturating_sub(1) .try_into() .unwrap_or(usize::MAX) .min(MAX_OVERFLOW_PAGES) .checked_mul(usable.checked_sub(4).ok_or("SQLite usable page size is too small")?) .ok_or("SQLite overflow capacity overflow")?; let capacity = local.checked_add(overflow_bytes).ok_or("SQLite payload capacity overflow")?.min(payload_len); let mut out = Vec::with_capacity(capacity); out.extend_from_slice(page.get(cell_at..local_end).ok_or("truncated SQLite cell payload")?); let mut remaining = payload_len.checked_sub(local).ok_or("SQLite payload length is too small")?; if remaining == 0 { return Ok(out); } let overflow_at = local_end; match overflow_at.checked_add(4) { Some(end) if end <= usable => {} _ => return Err("SQLite overflow pointer is in reserved page bytes".into()), } let first = be_u32_at(page, overflow_at)?; let mut next = first; let mut seen = HashSet::new(); while remaining > 0 { if next == 0 || !seen.insert(next) || seen.len() > MAX_OVERFLOW_PAGES { return Err("invalid or cyclic SQLite overflow chain".into()); } let overflow = self.page(next)?; let take = remaining.min(usable.checked_sub(4).ok_or("SQLite usable page size is too small")?); let end = 4usize.checked_add(take).ok_or("overflow payload offset")?; out.extend_from_slice(overflow.get(4..end).ok_or("truncated SQLite overflow page")?); remaining -= take; if remaining > 0 { next = be_u32_at(overflow, 0)?; } } Ok(out) } fn parse_schema(&self) -> Result { let mut tables = Vec::new(); let mut aliases = HashMap::new(); let mut state = WalkState { seen: HashSet::new(), out: Vec::new(), payload_total: 0 }; self.visit_table(1, 5, None, 0, &mut state)?; for record in state.out { if record.values.len() < 5 { return Err("sqlite_schema row has too few columns".into()); } let kind = match record.values.first() { Some(Value::Text(s)) => s.as_str(), _ => return Err("sqlite_schema row has invalid type".into()), }; let name = match record.values.get(1) { Some(Value::Text(s)) => s.clone(), _ => return Err("sqlite_schema row has invalid name".into()), }; if kind.eq_ignore_ascii_case("virtual") { continue; } if !kind.eq_ignore_ascii_case("table") { continue; } let rootpage = match record.values.get(3) { Some(Value::Integer(n)) if *n > 0 => u32::try_from(*n).map_err(|_| "sqlite_schema root page is invalid")?, _ => continue, }; let sql = match record.values.get(4) { Some(Value::Text(s)) => s, _ => continue, }; let Ok((column_names, alias)) = parse_create_table(sql) else { continue }; if column_names.len() > MAX_COLUMNS { continue; } if tables.iter().any(|table: &LiveTable| table.rootpage == rootpage) { continue; } if let Some(index) = alias { aliases.insert(rootpage, index); } tables.push(LiveTable { name, rootpage, column_names: Some(column_names) }); } Ok(SchemaInfo { tables, aliases }) } } fn validate_header_fields(header: &[u8], page_size: usize) -> Result { let write_version = *header.get(18).ok_or("truncated SQLite header")?; let read_version = *header.get(19).ok_or("truncated SQLite header")?; if !matches!(write_version, 1 | 2) || !matches!(read_version, 1 | 2) { return Err("unsupported SQLite file format version".into()); } if *header.get(21).ok_or("truncated SQLite header")? != 64 || *header.get(22).ok_or("truncated SQLite header")? != 32 || *header.get(23).ok_or("truncated SQLite header")? != 32 { return Err("invalid SQLite payload fractions".into()); } let reserved = usize::from(*header.get(20).ok_or("truncated SQLite header")?); let usable = page_size.checked_sub(reserved).ok_or("invalid SQLite reserved-byte count")?; if usable < 480 { return Err("SQLite usable page size is below 480 bytes".into()); } // 0 = no schema yet (file format §1.3.10): a catalog whose tables are all still in its WAL // has that in the main file, and the WAL's own page 1 is checked again with the overlay let schema_format = be_u32(header, 44)?; if !(0..=4).contains(&schema_format) { return Err("unsupported SQLite schema format".into()); } Ok(reserved) } fn parse_wal(wal: &[u8], page_size: usize) -> Result { let header = wal.get(..32).ok_or("truncated WAL header")?; let magic = be_u32(header, 0)?; if magic != 0x377f0682 && magic != 0x377f0683 { return Err("invalid WAL magic".into()); } if be_u32(header, 4)? != 3_007_000 || be_u32(header, 8)? != u32::try_from(page_size).map_err(|_| "page size overflow")? { return Err("invalid WAL page size or format version".into()); } let stride = page_size.checked_add(24).ok_or("WAL frame size overflow")?; let body = wal.get(32..).ok_or("truncated WAL body")?; if body.len() % stride != 0 { return Err("WAL ends with a partial frame".into()); } let salt = (be_u32(header, 16)?, be_u32(header, 20)?); let frame_count = body.len() / stride; if frame_count > MAX_CELLS { return Err("WAL has too many frames".into()); } let mut overlay = HashMap::new(); let mut page_count = None; for frame_no in 0..frame_count { let start = frame_no.checked_mul(stride).ok_or("WAL frame offset overflow")?; let end = start.checked_add(stride).ok_or("WAL frame offset overflow")?; let frame = body.get(start..end).ok_or("truncated WAL frame")?; let page_id = be_u32(frame, 0)?; if page_id == 0 || page_id > MAX_PAGE_ID || salt != (be_u32(frame, 8)?, be_u32(frame, 12)?) { return Err("invalid WAL frame page or salt".into()); } let commit = be_u32(frame, 4)?; if commit != 0 { if commit > MAX_PAGE_ID { return Err("WAL commit database size is too large".into()); } page_count = Some(commit); } overlay.insert(page_id, frame.get(24..).ok_or("truncated WAL page")?.to_vec()); } Ok(WalOverlay { page_size, frame_count, pages: overlay, page_count }) } fn decode_record(payload: &[u8], ncols: usize, encoding: TextEncoding) -> Result, String> { let mut at = 0; let header_len = usize_from_u64(read_varint(payload, &mut at)?, "record header length")?; if header_len < at || header_len > payload.len() { return Err("invalid SQLite record header length".into()); } let mut serials = Vec::new(); while at < header_len { if serials.len() >= ncols { return Err("SQLite record has more columns than table schema".into()); } serials.push(read_varint(payload, &mut at)?); } if at != header_len { return Err("SQLite record header is malformed".into()); } let mut body = header_len; let mut values = Vec::with_capacity(ncols); for serial in serials { let (value, size) = serial_value(payload, &mut body, serial, encoding)?; values.push(value); if values.len() > MAX_COLUMNS { return Err("SQLite record has too many columns".into()); } let _ = size; } if body > payload.len() { return Err("SQLite record body is truncated".into()); } values.resize(ncols, Value::Null); Ok(values) } fn serial_value(payload: &[u8], body: &mut usize, serial: u64, encoding: TextEncoding) -> Result<(Value, usize), String> { let (kind, len) = match serial { 0 => (0, 0), 1 => (1, 1), 2 => (2, 2), 3 => (3, 3), 4 => (4, 4), 5 => (5, 6), 6 => (6, 8), 7 => (7, 8), 8 => (8, 0), 9 => (9, 0), 10 | 11 => return Err("reserved SQLite serial type".into()), n if n >= 12 => { let len = usize_from_u64((n - if n % 2 == 0 { 12 } else { 13 }) / 2, "SQLite field length")?; (if n % 2 == 0 { 12 } else { 13 }, len) } _ => return Err("invalid SQLite serial type".into()), }; let end = body.checked_add(len).ok_or("SQLite record body offset overflow")?; let bytes = payload.get(*body..end).ok_or("SQLite record body is truncated")?; let value = match kind { 0 => Value::Null, 1 => Value::Integer(i64::from(i8::from_be_bytes([*bytes.first().ok_or("truncated integer")?]))), 2 => Value::Integer(i64::from(i16::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?))), 3 => { let first = *bytes.first().ok_or("truncated integer")?; let second = *bytes.get(1).ok_or("truncated integer")?; let third = *bytes.get(2).ok_or("truncated integer")?; let n = (u32::from(first) << 16) | (u32::from(second) << 8) | u32::from(third); let signed = if n & 0x80_0000 != 0 { n | 0xff00_0000 } else { n }; Value::Integer(i64::from(i32::from_be_bytes(signed.to_be_bytes()))) } 4 => Value::Integer(i64::from(i32::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?))), 5 => { let n = bytes.iter().fold(0u64, |acc, b| (acc << 8) | u64::from(*b)); let signed = if n & (1 << 47) != 0 { n | 0xffff_0000_0000_0000 } else { n }; Value::Integer(i64::from_be_bytes(signed.to_be_bytes())) } 6 => Value::Integer(i64::from_be_bytes(bytes.try_into().map_err(|_| "truncated integer")?)), 7 => { let n = f64::from_be_bytes(bytes.try_into().map_err(|_| "truncated real")?); if !n.is_finite() { return Err("non-finite SQLite real value".into()); } Value::Real(n) } 8 => Value::Integer(0), 9 => Value::Integer(1), 12 => Value::Blob(bytes.to_vec()), 13 => Value::Text(decode_text(bytes, encoding)?), _ => return Err("invalid SQLite field type".into()), }; *body = end; Ok((value, len)) } fn decode_text(bytes: &[u8], encoding: TextEncoding) -> Result { match encoding { TextEncoding::Utf8 => String::from_utf8(bytes.to_vec()).map_err(|_| "invalid UTF-8 SQLite text".into()), TextEncoding::Utf16Le | TextEncoding::Utf16Be => { if !bytes.len().is_multiple_of(2) { return Err("odd-length UTF-16 SQLite text".into()); } let units: Result, String> = bytes .as_chunks::<2>() .0 .iter() .map(|pair| { let first = *pair.first().ok_or("truncated UTF-16 SQLite text")?; let second = *pair.get(1).ok_or("truncated UTF-16 SQLite text")?; Ok(match encoding { TextEncoding::Utf16Le => u16::from_le_bytes([first, second]), TextEncoding::Utf16Be => u16::from_be_bytes([first, second]), TextEncoding::Utf8 => return Err("invalid UTF-16 encoding state".into()), }) }) .collect(); String::from_utf16(&units?).map_err(|_| "invalid UTF-16 SQLite text".into()) } } } fn parse_create_table(sql: &str) -> Result<(Vec, Option), String> { let tokens = tokenize_sql(sql)?; if !tokens.first().is_some_and(|t| t.word_is("CREATE")) { return Err("malformed CREATE TABLE SQL".into()); } let mut kind_at = 1usize; if tokens.get(kind_at).is_some_and(|t| t.word_is("TEMPORARY") || t.word_is("TEMP")) { kind_at += 1; } if tokens.get(kind_at).is_some_and(|t| t.word_is("VIRTUAL")) { return Err("virtual tables are unsupported".into()); } if !tokens.get(kind_at).is_some_and(|t| t.word_is("TABLE")) { return Err("malformed CREATE TABLE SQL".into()); } let open = tokens.iter().position(|t| t.is_symbol(b'(')).ok_or("CREATE TABLE has no column list")?; let mut depth = 0usize; let mut close = None; for (i, token) in tokens.iter().enumerate().skip(open) { if token.is_symbol(b'(') { depth += 1; } else if token.is_symbol(b')') { depth = depth.checked_sub(1).ok_or("unbalanced CREATE TABLE parentheses")?; if depth == 0 { close = Some(i); break; } } } let close = close.ok_or("unterminated CREATE TABLE column list")?; if tokens .get(close + 1..) .ok_or("invalid CREATE TABLE suffix")? .windows(2) .any(|w| w.first().is_some_and(|t| t.word_is("WITHOUT")) && w.get(1).is_some_and(|t| t.word_is("ROWID"))) { return Err("WITHOUT ROWID tables are unsupported".into()); } let body = tokens.get(open + 1..close).ok_or("invalid CREATE TABLE column list")?; let mut columns = Vec::new(); let mut alias = None; let mut integer_columns = HashSet::new(); let mut table_primary = None; let mut start = 0usize; let mut nested = 0usize; for i in 0..=body.len() { let comma = i == body.len() || (nested == 0 && body.get(i).is_some_and(|t| t.is_symbol(b','))); if comma { let part = body.get(start..i).ok_or("invalid CREATE TABLE column range")?; if part.is_empty() { return Err("empty CREATE TABLE column definition".into()); } if is_table_constraint(part) { if table_primary.is_none() { table_primary = table_primary_column(part); } } else { let name = part.first().and_then(Token::identifier).ok_or("column name is not an identifier")?; if columns.iter().any(|old| old == name) { return Err("duplicate SQLite column name".into()); } let index = columns.len(); if exact_integer_decl(part) { integer_columns.insert(name.to_ascii_lowercase()); } if integer_primary_key_alias(part) { alias = Some(index); } columns.push(name.to_string()); } start = i + 1; } else if body.get(i).is_some_and(|token| token.is_symbol(b'(')) { nested += 1; } else if body.get(i).is_some_and(|token| token.is_symbol(b')')) { nested = nested.checked_sub(1).ok_or("unbalanced CREATE TABLE column definition")?; } } if columns.is_empty() { return Err("CREATE TABLE has no columns".into()); } if alias.is_none() && let Some(primary) = table_primary { let key = primary.to_ascii_lowercase(); if integer_columns.contains(&key) { alias = columns.iter().position(|column| column.eq_ignore_ascii_case(&primary)); } } Ok((columns, alias)) } fn is_table_constraint(part: &[Token]) -> bool { part.first().is_some_and(|token| ["CONSTRAINT", "PRIMARY", "UNIQUE", "CHECK", "FOREIGN"].iter().any(|word| token.word_is(word))) } fn exact_integer_decl(part: &[Token]) -> bool { if !part.get(1).is_some_and(|token| token.word_is("INTEGER")) { return false; } match part.get(2) { None => true, Some(token) if token.is_symbol(b'(') => false, Some(token) => ["PRIMARY", "NOT", "NULL", "UNIQUE", "CHECK", "DEFAULT", "COLLATE", "REFERENCES", "CONSTRAINT", "GENERATED"] .iter() .any(|word| token.word_is(word)), } } fn table_primary_column(part: &[Token]) -> Option { let primary = part.iter().position(|token| token.word_is("PRIMARY"))?; if !part.get(primary + 1).is_some_and(|token| token.word_is("KEY")) { return None; } let open = primary + 2; if !part.get(open).is_some_and(|token| token.is_symbol(b'(')) { return None; } let close = (open + 1..part.len()).find(|index| part.get(*index).is_some_and(|token| token.is_symbol(b')')))?; let terms = part.get(open + 1..close)?; if terms.len() == 1 || (terms.len() == 2 && terms.get(1).is_some_and(|token| token.word_is("ASC") || token.word_is("DESC"))) { return terms.first().and_then(Token::identifier).map(str::to_owned); } None } fn integer_primary_key_alias(part: &[Token]) -> bool { if !exact_integer_decl(part) { return false; } let mut primary = None; let mut depth = 0usize; for i in 2..part.len() { if part.get(i).is_some_and(|token| token.is_symbol(b'(')) { depth += 1; } else if part.get(i).is_some_and(|token| token.is_symbol(b')')) { depth = depth.saturating_sub(1); } else if depth == 0 && part.get(i).is_some_and(|token| token.word_is("PRIMARY")) && part.get(i + 1).is_some_and(|t| t.word_is("KEY")) { primary = Some(i); break; } } let Some(i) = primary else { return false }; !part.get(i + 2).is_some_and(|token| token.word_is("DESC")) && !part.get(i + 2).is_some_and(|token| token.is_symbol(b'(')) } #[derive(Clone, Debug)] enum Token { Word(String), Quoted(String), StringLiteral, Symbol(u8), } impl Token { fn is_symbol(&self, symbol: u8) -> bool { matches!(self, Token::Symbol(value) if *value == symbol) } fn word_is(&self, wanted: &str) -> bool { matches!(self, Token::Word(word) if word.eq_ignore_ascii_case(wanted)) } fn identifier(&self) -> Option<&str> { match self { Token::Word(word) | Token::Quoted(word) => Some(word), _ => None, } } } fn tokenize_sql(sql: &str) -> Result, String> { let bytes = sql.as_bytes(); let mut out = Vec::new(); let mut at = 0usize; while at < bytes.len() { if bytes.get(at).is_some_and(u8::is_ascii_whitespace) { at += 1; } else if bytes.get(at) == Some(&b'-') && bytes.get(at + 1) == Some(&b'-') { at += 2; while at < bytes.len() && bytes.get(at) != Some(&b'\n') { at += 1; } } else if bytes.get(at) == Some(&b'/') && bytes.get(at + 1) == Some(&b'*') { at += 2; let mut closed = false; while at + 1 < bytes.len() { if bytes.get(at) == Some(&b'*') && bytes.get(at + 1) == Some(&b'/') { at += 2; closed = true; break; } at += 1; } if !closed { return Err("unterminated SQL comment".into()); } } else if matches!(bytes.get(at), Some(b'(' | b')' | b',' | b'.' | b';')) { out.push(Token::Symbol(*bytes.get(at).ok_or("truncated SQL punctuation")?)); at += 1; } else if matches!(bytes.get(at), Some(b'"' | b'`' | b'[')) { let quote = *bytes.get(at).ok_or("truncated SQL quote")?; let end_quote = if quote == b'[' { b']' } else { quote }; at += 1; let mut value = Vec::new(); let mut closed = false; while at < bytes.len() { if bytes.get(at) == Some(&end_quote) { if bytes.get(at + 1) == Some(&end_quote) { value.push(end_quote); at += 2; } else { at += 1; closed = true; break; } } else { value.push(*bytes.get(at).ok_or("truncated SQL identifier")?); at += 1; } } if !closed { return Err("unterminated quoted SQL identifier".into()); } out.push(Token::Quoted(String::from_utf8(value).map_err(|_| "invalid SQL identifier encoding")?)); } else if bytes.get(at) == Some(&b'\'') { at += 1; let mut closed = false; while at < bytes.len() { if bytes.get(at) == Some(&b'\'') { if bytes.get(at + 1) == Some(&b'\'') { at += 2; } else { at += 1; closed = true; break; } } else { at += 1; } } if !closed { return Err("unterminated SQL string".into()); } out.push(Token::StringLiteral); } else { let start = at; while at < bytes.len() && !bytes.get(at).is_some_and(u8::is_ascii_whitespace) && !matches!(bytes.get(at), Some(b'(' | b')' | b',' | b'.' | b';' | b'\'' | b'"' | b'`' | b'[')) { at += 1; } if start == at { return Err("unsupported SQL punctuation".into()); } out.push(Token::Word(sql.get(start..at).ok_or("invalid SQL token boundary")?.to_string())); } } Ok(out) } fn sqlite_page_size(value: u16) -> Result { let size = if value == 1 { 65_536 } else { usize::from(value) }; if !(512..=65_536).contains(&size) || !size.is_power_of_two() { return Err("invalid SQLite page size".into()); } Ok(size) } fn read_varint(bytes: &[u8], at: &mut usize) -> Result { let mut value = 0u64; for i in 0..9 { let byte = *bytes.get(*at).ok_or("truncated SQLite varint")?; *at = (*at).checked_add(1).ok_or("SQLite varint offset overflow")?; if i == 8 { value = (value << 8) | u64::from(byte); return Ok(value); } value = (value << 7) | u64::from(byte & 0x7f); if byte & 0x80 == 0 { return Ok(value); } } Err("invalid SQLite varint".into()) } fn usize_from_u64(value: u64, label: &str) -> Result { usize::try_from(value).map_err(|_| format!("{label} exceeds host size")) } fn be_u16(bytes: &[u8], at: usize) -> Result { let end = at.checked_add(2).ok_or("SQLite integer offset overflow")?; bytes.get(at..end).and_then(|s| <[u8; 2]>::try_from(s).ok()).map(u16::from_be_bytes).ok_or("truncated SQLite integer".into()) } fn be_u16_at(bytes: &[u8], at: usize) -> Result { be_u16(bytes, at) } fn be_u32(bytes: &[u8], at: usize) -> Result { let end = at.checked_add(4).ok_or("SQLite integer offset overflow")?; bytes.get(at..end).and_then(|s| <[u8; 4]>::try_from(s).ok()).map(u32::from_be_bytes).ok_or("truncated SQLite integer".into()) } fn be_u32_at(bytes: &[u8], at: usize) -> Result { be_u32(bytes, at) } #[cfg(test)] mod tests { use super::*; fn varint(mut value: u64) -> Vec { let mut out = vec![u8::try_from(value & 0x7f).unwrap_or(0)]; value >>= 7; while value != 0 { out.push(u8::try_from(value & 0x7f).unwrap_or(0) | 0x80); value >>= 7; } out.reverse(); out } fn leaf(page: &mut [u8], base: usize, rowid: u64, payload: &[u8]) { page[base] = 0x0d; page[base + 3..base + 5].copy_from_slice(&1u16.to_be_bytes()); let mut cell = varint(payload.len() as u64); cell.extend(varint(rowid)); cell.extend(payload); let start = page.len() - cell.len(); page[start..].copy_from_slice(&cell); page[base + 8..base + 10].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); page[base + 5..base + 7].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); } fn leaf_rows(page: &mut [u8], base: usize, rows: &[(u64, &[u8])]) { page[base] = 0x0d; page[base + 3..base + 5].copy_from_slice(&u16::try_from(rows.len()).unwrap_or(0).to_be_bytes()); let mut end = page.len(); for (index, (rowid, payload)) in rows.iter().enumerate() { let mut cell = varint(payload.len() as u64); cell.extend(varint(*rowid)); cell.extend(*payload); end -= cell.len(); page[end..end + cell.len()].copy_from_slice(&cell); let pointer = base + 8 + index * 2; page[pointer..pointer + 2].copy_from_slice(&u16::try_from(end).unwrap_or(0).to_be_bytes()); } page[base + 5..base + 7].copy_from_slice(&u16::try_from(end).unwrap_or(0).to_be_bytes()); } fn schema_record(name: &str, root: u8, sql: &str) -> Vec { let mut header = vec![23]; let text_serial = |text: &str| 13 + 2 * text.len() as u64; header.extend(varint(text_serial(name))); header.extend(varint(text_serial(name))); header.push(1); header.extend(varint(text_serial(sql))); let mut body = b"table".to_vec(); body.extend(name.as_bytes()); body.extend(name.as_bytes()); body.push(root); body.extend(sql.as_bytes()); let mut record = varint((1 + header.len()) as u64); record.extend(header); record.extend(body); record } #[test] fn schema_names_and_integer_primary_key_alias_are_decoded() { let sql = r#"CREATE TABLE "t" ("id_local" INTEGER PRIMARY KEY, "odd,name" TEXT, PRIMARY KEY ("id_local"))"#; let (names, alias) = parse_create_table(sql).unwrap(); assert_eq!(names, vec!["id_local", "odd,name"]); assert_eq!(alias, Some(0)); let (_, no_alias) = parse_create_table("CREATE TABLE t (id INTEGER PRIMARY KEY DESC, value TEXT)").unwrap(); assert_eq!(no_alias, None); let (table_names, table_alias) = parse_create_table("CREATE TABLE t (id INTEGER, value TEXT, PRIMARY KEY(id))").unwrap(); assert_eq!(table_names, vec!["id", "value"]); assert_eq!(table_alias, Some(0)); } /// A database file with no schema yet has schema format 0 in its header (file format /// §1.3.10): a catalog created in WAL mode whose tables are all still in its `-wal` file (one /// Lightroom has open and hasn't checkpointed) was refused as "unsupported SQLite schema /// format" before the WAL was looked at. It opens, with no tables; other formats stay refused. #[test] fn schema_format_zero_is_an_empty_database() { let header = |format: u32| { let mut main = vec![0u8; 512]; main[..16].copy_from_slice(b"SQLite format 3\0"); main[16..18].copy_from_slice(&512u16.to_be_bytes()); main[18] = 2; main[19] = 2; main[21] = 64; main[22] = 32; main[23] = 32; main[28..32].copy_from_slice(&1u32.to_be_bytes()); main[44..48].copy_from_slice(&format.to_be_bytes()); main[56..60].copy_from_slice(&1u32.to_be_bytes()); main[100] = 0x0d; main[105..107].copy_from_slice(&512u16.to_be_bytes()); main }; let db = Database::open_with_wal(header(0), &[]).unwrap(); assert!(db.live_tables().unwrap().is_empty()); assert!(Database::open_with_wal(header(4), &[]).is_ok()); assert_eq!(Database::open_with_wal(header(5), &[]).err().as_deref(), Some("unsupported SQLite schema format")); } #[test] fn unsupported_unrelated_schema_entries_are_skipped() { let mut main = vec![0u8; 2 * 512]; main[..16].copy_from_slice(b"SQLite format 3\0"); main[16..18].copy_from_slice(&512u16.to_be_bytes()); main[18] = 1; main[19] = 1; main[21] = 64; main[22] = 32; main[23] = 32; main[28..32].copy_from_slice(&2u32.to_be_bytes()); main[44..48].copy_from_slice(&4u32.to_be_bytes()); main[56..60].copy_from_slice(&1u32.to_be_bytes()); let virtual_sql = "CREATE VIRTUAL TABLE unrelated USING fts5(text)"; let supported_sql = "CREATE TABLE supported (id INTEGER PRIMARY KEY)"; let virtual_record = schema_record("unrelated", 2, virtual_sql); let supported_record = schema_record("supported", 2, supported_sql); leaf_rows(&mut main[..512], 100, &[(1, &virtual_record), (2, &supported_record)]); let db = Database::open_with_wal(main, &[]).unwrap(); let tables = db.live_tables().unwrap(); assert_eq!(tables.len(), 1); assert_eq!(tables[0].name, "supported"); } #[test] fn synthetic_btree_substitutes_rowid_and_reads_schema() { let mut main = vec![0u8; 2 * 512]; main[..16].copy_from_slice(b"SQLite format 3\0"); main[16..18].copy_from_slice(&512u16.to_be_bytes()); main[18] = 1; main[19] = 1; main[21] = 64; main[22] = 32; main[23] = 32; main[28..32].copy_from_slice(&2u32.to_be_bytes()); main[44..48].copy_from_slice(&4u32.to_be_bytes()); main[56..60].copy_from_slice(&1u32.to_be_bytes()); let sql = "CREATE TABLE \"t\" (\"id_local\" INTEGER PRIMARY KEY, \"name\" TEXT)"; // rootpage is an integer field, so replace fourth text value with a compact record. let mut header = vec![23, 15, 15, 1]; header.extend(varint((13 + 2 * sql.len()) as u64)); let mut body = b"tablett".to_vec(); body.push(2); body.extend(sql.as_bytes()); let mut record = varint((1 + header.len()) as u64); record.append(&mut header); record.extend(body); let schema = record; leaf(&mut main[512..], 0, 1, &[3, 0, 15, b'x']); leaf(&mut main[..512], 100, 1, &schema); let db = Database::open_with_wal(main, &[]).unwrap(); let tables = db.live_tables().unwrap(); assert!(tables.iter().any(|table| table.name == "t")); let rows = db.read_table(2, 2).unwrap(); assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(1))); } #[test] fn interior_table_pointer_array_uses_cell_offsets() { let mut main = vec![0u8; 4 * 512]; let mut interior = vec![0u8; 512]; interior[0] = 0x05; interior[3..5].copy_from_slice(&1u16.to_be_bytes()); interior[8..12].copy_from_slice(&4u32.to_be_bytes()); interior[12..14].copy_from_slice(&500u16.to_be_bytes()); interior[500..504].copy_from_slice(&3u32.to_be_bytes()); interior[504] = 1; main[512..1024].copy_from_slice(&interior); leaf(&mut main[1024..1536], 0, 1, &[2, 0]); leaf(&mut main[1536..], 0, 2, &[2, 0]); let db = Database { main, overlay: HashMap::new(), page_size: 512, reserved: 0, encoding: TextEncoding::Utf8, page_count: 4, tables: Vec::new(), rowid_aliases: HashMap::new(), }; assert_eq!(db.read_table(2, 1).unwrap().len(), 2); } #[test] fn negative_rowid_varint_is_signed() { let mut main = vec![0u8; 512]; let page = &mut main[..]; page[100] = 0x0d; page[103..105].copy_from_slice(&1u16.to_be_bytes()); let start = 500usize; page[start] = 2; for offset in 1..10 { if let Some(byte) = page.get_mut(start + offset) { *byte = 0xff; } } page[start + 10..start + 12].copy_from_slice(&[2, 0]); page[108..110].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); page[105..107].copy_from_slice(&u16::try_from(start).unwrap_or(0).to_be_bytes()); let mut aliases = HashMap::new(); aliases.insert(1, 0); let db = Database { main, overlay: HashMap::new(), page_size: 512, reserved: 0, encoding: TextEncoding::Utf8, page_count: 1, tables: Vec::new(), rowid_aliases: aliases, }; let rows = db.read_table(1, 1).unwrap(); assert_eq!(rows.first().and_then(|row| row.values.first()), Some(&Value::Integer(-1))); } #[test] fn reserved_page_trailer_cannot_supply_cells() { let mut main = vec![0u8; 512]; main[100] = 0x0d; main[103..105].copy_from_slice(&1u16.to_be_bytes()); main[108..110].copy_from_slice(&490u16.to_be_bytes()); let db = Database { main, overlay: HashMap::new(), page_size: 512, reserved: 32, encoding: TextEncoding::Utf8, page_count: 1, tables: Vec::new(), rowid_aliases: HashMap::new(), }; assert!(db.read_table(1, 1).is_err()); } #[test] fn utf16_and_overflow_payloads_are_supported() { let mut payload = vec![2, 17, b'A', 0]; assert_eq!(decode_record(&payload, 1, TextEncoding::Utf16Le).unwrap(), vec![Value::Text("A".into())]); payload.clear(); payload.extend([2, 17, 0, b'A']); assert_eq!(decode_record(&payload, 1, TextEncoding::Utf16Be).unwrap(), vec![Value::Text("A".into())]); let mut main = vec![0u8; 2 * 512]; let db = Database { main: std::mem::take(&mut main), overlay: HashMap::new(), page_size: 512, reserved: 0, encoding: TextEncoding::Utf8, page_count: 2, tables: Vec::new(), rowid_aliases: HashMap::new(), }; let mut cell_page = vec![b'x'; 512]; // For a 512-byte table leaf & 600-byte payload, SQLite stores 92 // payload bytes locally, followed by the overflow page pointer. cell_page[92..96].copy_from_slice(&2u32.to_be_bytes()); let inline_page = vec![b'i'; 512]; assert_eq!(db.payload(&inline_page, 0, 110).unwrap(), vec![b'i'; 110]); let mut overflow_page = vec![b'y'; 512]; overflow_page[..4].copy_from_slice(&0u32.to_be_bytes()); let mut db = db; db.main[512..].copy_from_slice(&overflow_page); let mut with_overflow = db.overlay; with_overflow.insert(2, overflow_page); db.overlay = with_overflow; let bytes = db.payload(&cell_page, 0, 600).unwrap(); assert_eq!(bytes.len(), 600); assert_eq!(&bytes[..92], &[b'x'; 92]); assert_eq!(&bytes[92..], &[b'y'; 508]); } #[test] fn malformed_varint_and_wal_cycle_fail_cleanly() { let mut at = 0; assert!(read_varint(&[0x80; 8], &mut at).is_err()); let mut wal = vec![0u8; 32 + 24 + 512]; wal[..4].copy_from_slice(&0x377f0682u32.to_be_bytes()); wal[4..8].copy_from_slice(&3_007_000u32.to_be_bytes()); wal[8..12].copy_from_slice(&512u32.to_be_bytes()); wal[16..20].copy_from_slice(&7u32.to_be_bytes()); wal[20..24].copy_from_slice(&8u32.to_be_bytes()); wal[32..36].copy_from_slice(&1u32.to_be_bytes()); wal[36..40].copy_from_slice(&1u32.to_be_bytes()); wal[40..44].copy_from_slice(&7u32.to_be_bytes()); wal[44..48].copy_from_slice(&8u32.to_be_bytes()); let parsed = parse_wal(&wal, 512).unwrap(); assert_eq!(parsed.frame_count, 1); assert_eq!(parsed.page_count, Some(1)); assert_eq!(parsed.pages.get(&1).map(Vec::len), Some(512)); let mut page = vec![0u8; 512]; page[16..18].copy_from_slice(&512u16.to_be_bytes()); let db = Database { main: vec![0; 512], overlay: HashMap::new(), page_size: 512, reserved: 0, encoding: TextEncoding::Utf8, page_count: 1, tables: Vec::new(), rowid_aliases: HashMap::new(), }; assert!(db.payload(&page, 0, 600).is_err()); } }