From def11a1dfb9566cafabe6bbd2f9fa9241dad040b Mon Sep 17 00:00:00 2001 From: Till Wegmueller Date: Wed, 4 Feb 2026 22:39:42 +0100 Subject: [PATCH] Refactor: Replace `redb` with `rusqlite` for catalog handling - Transitioned the catalog backend from `redb` to `rusqlite` for better compatibility and concurrency. - Updated `IpsProvider` to use SQLite for package querying, dependency resolution, and obsolescence handling. - Removed `decode_manifest_bytes_local` and unused `manifest` cache logic. - Simplified catalog-related functions and integrated `sqlite_catalog` module. - Enhanced test functions and added schemas for managing SQLite databases. --- Cargo.lock | 114 +- libips/Cargo.toml | 9 +- libips/src/image/catalog.rs | 1207 ++++------------------ libips/src/image/installed.rs | 374 ++----- libips/src/image/installed_tests.rs | 18 +- libips/src/image/mod.rs | 174 +++- libips/src/repository/file_backend.rs | 43 +- libips/src/repository/mod.rs | 49 +- libips/src/repository/obsoleted.rs | 672 +++++------- libips/src/repository/shard_sync.rs | 164 +++ libips/src/repository/sqlite_catalog.rs | 484 +++++++++ libips/src/solver/mod.rs | 435 +++----- pkg6depotd/Cargo.toml | 6 + pkg6depotd/src/http/handlers/mod.rs | 1 + pkg6depotd/src/http/handlers/shard.rs | 127 +++ pkg6depotd/src/http/handlers/versions.rs | 2 +- pkg6depotd/src/http/routes.rs | 10 +- pkg6depotd/src/repo.rs | 7 + 18 files changed, 1790 insertions(+), 2106 deletions(-) create mode 100644 libips/src/repository/shard_sync.rs create mode 100644 libips/src/repository/sqlite_catalog.rs create mode 100644 pkg6depotd/src/http/handlers/shard.rs diff --git a/Cargo.lock b/Cargo.lock index 01b1bcd..edbb8bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -326,26 +326,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "bincode" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740" -dependencies = [ - "bincode_derive", - "serde", - "unty", -] - -[[package]] -name = "bincode_derive" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09" -dependencies = [ - "virtue", -] - [[package]] name = "bitflags" version = "2.10.0" @@ -765,6 +745,18 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.3.0" @@ -1020,12 +1012,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "half" -version = "1.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403" - [[package]] name = "hashbrown" version = "0.14.5" @@ -1051,6 +1037,15 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + [[package]] name = "heck" version = "0.4.1" @@ -1463,7 +1458,7 @@ checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" name = "libips" version = "0.5.3" dependencies = [ - "bincode", + "base64 0.22.1", "chrono", "diff-struct", "flate2", @@ -1474,14 +1469,13 @@ dependencies = [ "object", "pest", "pest_derive", - "redb", "regex", "reqwest", "resolvo", + "rusqlite", "rust-ini", "semver", "serde", - "serde_cbor", "serde_json", "sha1", "sha2", @@ -1503,6 +1497,17 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -2120,10 +2125,12 @@ dependencies = [ "opentelemetry_sdk", "predicates", "reqwest", + "rusqlite", "rustls", "serde", "serde_json", "sha1", + "sha2", "socket2", "tempfile", "thiserror 2.0.17", @@ -2197,7 +2204,7 @@ dependencies = [ "reqwest", "shellexpand", "specfile", - "thiserror 1.0.69", + "thiserror 2.0.17", "url", "which", ] @@ -2411,15 +2418,6 @@ dependencies = [ "getrandom 0.3.4", ] -[[package]] -name = "redb" -version = "3.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae323eb086579a3769daa2c753bb96deb95993c534711e0dbe881b5192906a06" -dependencies = [ - "libc", -] - [[package]] name = "redox_syscall" version = "0.5.18" @@ -2548,6 +2546,20 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rusqlite" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rust-ini" version = "0.21.3" @@ -2729,16 +2741,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "serde_cbor" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" -dependencies = [ - "half", - "serde", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -2895,7 +2897,7 @@ dependencies = [ "anyhow", "pest", "pest_derive", - "thiserror 1.0.69", + "thiserror 2.0.17", ] [[package]] @@ -3479,12 +3481,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" -[[package]] -name = "unty" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae" - [[package]] name = "url" version = "2.5.7" @@ -3554,12 +3550,6 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" -[[package]] -name = "virtue" -version = "0.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" - [[package]] name = "wait-timeout" version = "0.2.1" diff --git a/libips/Cargo.toml b/libips/Cargo.toml index bd3a2ca..8d57fd0 100644 --- a/libips/Cargo.toml +++ b/libips/Cargo.toml @@ -32,20 +32,19 @@ pest_derive = "2.1.0" strum = { version = "0.27", features = ["derive"] } serde = { version = "1.0.207", features = ["derive"] } serde_json = "1.0.124" -serde_cbor = "0.11.2" flate2 = "1.0.28" lz4 = "1.24.0" +base64 = "0.22" semver = { version = "1.0.20", features = ["serde"] } diff-struct = "0.5.3" chrono = "0.4.41" tempfile = "3.20.0" walkdir = "2.4.0" -redb = { version = "3" } -bincode = { version = "2", features = ["serde"] } +rusqlite = { version = "0.31", default-features = false } rust-ini = "0.21" reqwest = { version = "0.12", features = ["blocking", "json", "gzip", "deflate"] } resolvo = "0.10" [features] -default = ["redb-index"] -redb-index = [] # Enable redb-based index for obsoleted packages +default = ["bundled-sqlite"] +bundled-sqlite = ["rusqlite/bundled"] diff --git a/libips/src/image/catalog.rs b/libips/src/image/catalog.rs index 8e49361..325f2a9 100644 --- a/libips/src/image/catalog.rs +++ b/libips/src/image/catalog.rs @@ -3,7 +3,6 @@ use crate::fmri::Fmri; use crate::repository::catalog::{CatalogManager, CatalogPart, PackageVersionEntry}; use lz4::{Decoder as Lz4Decoder, EncoderBuilder as Lz4EncoderBuilder}; use miette::Diagnostic; -use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fs; @@ -12,21 +11,6 @@ use std::path::{Path, PathBuf}; use thiserror::Error; use tracing::{info, trace, warn}; -/// Table definition for the catalog database -/// Key: stem@version -/// Value: serialized Manifest -pub const CATALOG_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("catalog"); - -/// Table definition for the obsoleted packages catalog -/// Key: full FMRI including publisher (pkg://publisher/stem@version) -/// Value: nothing -pub const OBSOLETED_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("obsoleted"); - -/// Table definition for the incorporate locks table -/// Key: stem (e.g., "compress/gzip") -/// Value: version string as bytes (same format as Fmri::version()) -pub const INCORPORATE_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("incorporate"); - /// Errors that can occur when working with the image catalog #[derive(Error, Debug, Diagnostic)] pub enum CatalogError { @@ -74,7 +58,7 @@ fn is_likely_json(bytes: &[u8]) -> bool { matches!(bytes[i], b'{' | b'[') } -fn compress_json_lz4(bytes: &[u8]) -> Result> { +pub(crate) fn compress_json_lz4(bytes: &[u8]) -> Result> { let mut dst = Vec::with_capacity(bytes.len() / 2 + 32); let mut enc = Lz4EncoderBuilder::new() .level(4) @@ -87,7 +71,7 @@ fn compress_json_lz4(bytes: &[u8]) -> Result> { Ok(dst) } -fn decode_manifest_bytes(bytes: &[u8]) -> Result { +pub(crate) fn decode_manifest_bytes(bytes: &[u8]) -> Result { // Fast path: uncompressed legacy JSON if is_likely_json(bytes) { return Ok(serde_json::from_slice::(bytes)?); @@ -108,6 +92,13 @@ fn decode_manifest_bytes(bytes: &[u8]) -> Result { Ok(serde_json::from_slice::(&out)?) } +/// Check if a package manifest is marked as obsolete. +pub(crate) fn is_package_obsolete(manifest: &Manifest) -> bool { + manifest.attributes.iter().any(|attr| { + attr.key == "pkg.obsolete" && attr.values.first().map_or(false, |v| v == "true") + }) +} + /// Information about a package in the catalog #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PackageInfo { @@ -142,335 +133,28 @@ impl ImageCatalog { } } - /// Dump the contents of a specific table to stdout for debugging - pub fn dump_table(&self, table_name: &str) -> Result<()> { - // Determine which table to dump and open the appropriate database - match table_name { - "catalog" => { - let db = Database::open(&self.db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open catalog database: {}", e)) - })?; - let tx = db.begin_read().map_err(|e| { - CatalogError::Database(format!("Failed to begin transaction: {}", e)) - })?; - self.dump_catalog_table(&tx)?; - } - "obsoleted" => { - let db = Database::open(&self.obsoleted_db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open obsoleted database: {}", e)) - })?; - let tx = db.begin_read().map_err(|e| { - CatalogError::Database(format!("Failed to begin transaction: {}", e)) - })?; - self.dump_obsoleted_table(&tx)?; - } - "incorporate" => { - let db = Database::open(&self.db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open catalog database: {}", e)) - })?; - let tx = db.begin_read().map_err(|e| { - CatalogError::Database(format!("Failed to begin transaction: {}", e)) - })?; - // Simple dump of incorporate locks - if let Ok(table) = tx.open_table(INCORPORATE_TABLE) { - for entry in table.iter().map_err(|e| { - CatalogError::Database(format!( - "Failed to iterate incorporate table: {}", - e - )) - })? { - let (k, v) = entry.map_err(|e| { - CatalogError::Database(format!( - "Failed to read incorporate table entry: {}", - e - )) - })?; - let stem = k.value(); - let ver = String::from_utf8_lossy(v.value()); - println!("{} -> {}", stem, ver); - } - } - } - _ => { - return Err(CatalogError::Database(format!( - "Unknown table: {}", - table_name - ))); - } - } - - Ok(()) - } - - /// Dump the contents of all tables to stdout for debugging - pub fn dump_all_tables(&self) -> Result<()> { - // Catalog DB - let db_cat = Database::open(&self.db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open catalog database: {}", e)) - })?; - let tx_cat = db_cat - .begin_read() - .map_err(|e| CatalogError::Database(format!("Failed to begin transaction: {}", e)))?; - println!("=== CATALOG TABLE ==="); - let _ = self.dump_catalog_table(&tx_cat); - - // Obsoleted DB - let db_obs = Database::open(&self.obsoleted_db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open obsoleted database: {}", e)) - })?; - let tx_obs = db_obs - .begin_read() - .map_err(|e| CatalogError::Database(format!("Failed to begin transaction: {}", e)))?; - println!("\n=== OBSOLETED TABLE ==="); - let _ = self.dump_obsoleted_table(&tx_obs); - - Ok(()) - } - - /// Dump the contents of the catalog table - fn dump_catalog_table(&self, tx: &redb::ReadTransaction) -> Result<()> { - match tx.open_table(CATALOG_TABLE) { - Ok(table) => { - let mut count = 0; - for entry_result in table.iter().map_err(|e| { - CatalogError::Database(format!("Failed to iterate catalog table: {}", e)) - })? { - let (key, value) = entry_result.map_err(|e| { - CatalogError::Database(format!( - "Failed to get entry from catalog table: {}", - e - )) - })?; - let key_str = key.value(); - - // Try to deserialize the manifest (supports JSON or LZ4-compressed JSON) - match decode_manifest_bytes(value.value()) { - Ok(manifest) => { - // Extract the publisher from the FMRI attribute - let publisher = manifest - .attributes - .iter() - .find(|attr| attr.key == "pkg.fmri") - .and_then(|attr| attr.values.first().cloned()) - .unwrap_or_else(|| "unknown".to_string()); - - println!("Key: {}", key_str); - println!(" FMRI: {}", publisher); - println!(" Attributes: {}", manifest.attributes.len()); - println!(" Files: {}", manifest.files.len()); - println!(" Directories: {}", manifest.directories.len()); - println!(" Dependencies: {}", manifest.dependencies.len()); - } - Err(e) => { - println!("Key: {}", key_str); - println!(" Error deserializing manifest: {}", e); - } - } - count += 1; - } - println!("Total entries in catalog table: {}", count); - Ok(()) - } - Err(e) => { - println!("Error opening catalog table: {}", e); - Err(CatalogError::Database(format!( - "Failed to open catalog table: {}", - e - ))) - } - } - } - - /// Dump the contents of the obsoleted table - fn dump_obsoleted_table(&self, tx: &redb::ReadTransaction) -> Result<()> { - match tx.open_table(OBSOLETED_TABLE) { - Ok(table) => { - let mut count = 0; - for entry_result in table.iter().map_err(|e| { - CatalogError::Database(format!("Failed to iterate obsoleted table: {}", e)) - })? { - let (key, _) = entry_result.map_err(|e| { - CatalogError::Database(format!( - "Failed to get entry from obsoleted table: {}", - e - )) - })?; - let key_str = key.value(); - - println!("Key: {}", key_str); - count += 1; - } - println!("Total entries in obsoleted table: {}", count); - Ok(()) - } - Err(e) => { - println!("Error opening obsoleted table: {}", e); - Err(CatalogError::Database(format!( - "Failed to open obsoleted table: {}", - e - ))) - } - } - } - - /// Get database statistics - pub fn get_db_stats(&self) -> Result<()> { - // Open the catalog database - let db_cat = Database::open(&self.db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open catalog database: {}", e)) - })?; - let tx_cat = db_cat - .begin_read() - .map_err(|e| CatalogError::Database(format!("Failed to begin transaction: {}", e)))?; - - // Open the obsoleted database - let db_obs = Database::open(&self.obsoleted_db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open obsoleted database: {}", e)) - })?; - let tx_obs = db_obs - .begin_read() - .map_err(|e| CatalogError::Database(format!("Failed to begin transaction: {}", e)))?; - - // Get table statistics - let mut catalog_count = 0; - let mut obsoleted_count = 0; - - // Count catalog entries - if let Ok(table) = tx_cat.open_table(CATALOG_TABLE) { - for result in table.iter().map_err(|e| { - CatalogError::Database(format!("Failed to iterate catalog table: {}", e)) - })? { - let _ = result.map_err(|e| { - CatalogError::Database(format!("Failed to get entry from catalog table: {}", e)) - })?; - catalog_count += 1; - } - } - - // Count obsoleted entries (separate DB) - if let Ok(table) = tx_obs.open_table(OBSOLETED_TABLE) { - for result in table.iter().map_err(|e| { - CatalogError::Database(format!("Failed to iterate obsoleted table: {}", e)) - })? { - let _ = result.map_err(|e| { - CatalogError::Database(format!( - "Failed to get entry from obsoleted table: {}", - e - )) - })?; - obsoleted_count += 1; - } - } - - // Print statistics - println!("Catalog database path: {}", self.db_path.display()); - println!( - "Obsoleted database path: {}", - self.obsoleted_db_path.display() - ); - println!("Catalog directory: {}", self.catalog_dir.display()); - println!("Table statistics:"); - println!(" Catalog table: {} entries", catalog_count); - println!(" Obsoleted table: {} entries", obsoleted_count); - println!("Total entries: {}", catalog_count + obsoleted_count); - - Ok(()) - } - - /// Initialize the catalog database - pub fn init_db(&self) -> Result<()> { - // Ensure parent directories exist - if let Some(parent) = self.db_path.parent() { - fs::create_dir_all(parent)?; - } - if let Some(parent) = self.obsoleted_db_path.parent() { - fs::create_dir_all(parent)?; - } - - // Create/open catalog database and tables - let db_cat = Database::create(&self.db_path).map_err(|e| { - CatalogError::Database(format!("Failed to create catalog database: {}", e)) - })?; - let tx_cat = db_cat - .begin_write() - .map_err(|e| CatalogError::Database(format!("Failed to begin transaction: {}", e)))?; - tx_cat.open_table(CATALOG_TABLE).map_err(|e| { - CatalogError::Database(format!("Failed to create catalog table: {}", e)) - })?; - tx_cat.open_table(INCORPORATE_TABLE).map_err(|e| { - CatalogError::Database(format!("Failed to create incorporate table: {}", e)) - })?; - tx_cat.commit().map_err(|e| { - CatalogError::Database(format!("Failed to commit catalog transaction: {}", e)) - })?; - - // Create/open obsoleted database and table - let db_obs = Database::create(&self.obsoleted_db_path).map_err(|e| { - CatalogError::Database(format!("Failed to create obsoleted database: {}", e)) - })?; - let tx_obs = db_obs - .begin_write() - .map_err(|e| CatalogError::Database(format!("Failed to begin transaction: {}", e)))?; - tx_obs.open_table(OBSOLETED_TABLE).map_err(|e| { - CatalogError::Database(format!("Failed to create obsoleted table: {}", e)) - })?; - tx_obs.commit().map_err(|e| { - CatalogError::Database(format!("Failed to commit obsoleted transaction: {}", e)) - })?; - - Ok(()) - } - /// Build the catalog from downloaded catalogs + /// + /// This method is deprecated in favor of server-side shard building. + /// For client-side catalog updates, use shard_sync instead. pub fn build_catalog(&self, publishers: &[String]) -> Result<()> { - info!("Building catalog (publishers: {})", publishers.len()); - trace!("Catalog directory: {:?}", self.catalog_dir); - trace!("Catalog database path: {:?}", self.db_path); + use tracing::{info, warn}; + + info!("Building catalog shards (publishers: {})", publishers.len()); if publishers.is_empty() { return Err(CatalogError::NoPublishers); } - // Open the databases - trace!( - "Opening databases at {:?} and {:?}", - self.db_path, self.obsoleted_db_path - ); - let db_cat = Database::open(&self.db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open catalog database: {}", e)) - })?; - let db_obs = Database::open(&self.obsoleted_db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open obsoleted database: {}", e)) - })?; - - // Begin writing transactions - trace!("Beginning write transactions"); - let tx_cat = db_cat.begin_write().map_err(|e| { - CatalogError::Database(format!("Failed to begin catalog transaction: {}", e)) - })?; - let tx_obs = db_obs.begin_write().map_err(|e| { - CatalogError::Database(format!("Failed to begin obsoleted transaction: {}", e)) - })?; - - // Open the catalog table - trace!("Opening catalog table"); - let mut catalog_table = tx_cat - .open_table(CATALOG_TABLE) - .map_err(|e| CatalogError::Database(format!("Failed to open catalog table: {}", e)))?; - - // Open the obsoleted table - trace!("Opening obsoleted table"); - let mut obsoleted_table = tx_obs.open_table(OBSOLETED_TABLE).map_err(|e| { - CatalogError::Database(format!("Failed to open obsoleted table: {}", e)) + // Get the output directory for shards (parent of db_path) + let shard_dir = self.db_path.parent().ok_or_else(|| { + CatalogError::Database("Invalid database path - no parent directory".to_string()) })?; // Process each publisher for publisher in publishers { - trace!("Processing publisher: {}", publisher); let publisher_catalog_dir = self.catalog_dir.join(publisher); - trace!("Publisher catalog directory: {:?}", publisher_catalog_dir); - // Skip if the publisher catalog directory doesn't exist if !publisher_catalog_dir.exists() { warn!( "Publisher catalog directory not found: {}", @@ -479,8 +163,7 @@ impl ImageCatalog { continue; } - // Determine where catalog parts live. Support both legacy nested layout - // (publisher//catalog) and flat layout (directly under publisher dir). + // Determine where catalog parts live let nested_dir = publisher_catalog_dir .join("publisher") .join(publisher) @@ -488,15 +171,11 @@ impl ImageCatalog { let flat_dir = publisher_catalog_dir.clone(); let catalog_parts_dir = if nested_dir.exists() { - &nested_dir + nested_dir } else { - &flat_dir + flat_dir }; - trace!("Creating catalog manager for publisher: {}", publisher); - trace!("Catalog parts directory: {:?}", catalog_parts_dir); - - // Check if the catalog parts directory exists (either layout) if !catalog_parts_dir.exists() { warn!( "Catalog parts directory not found: {}", @@ -505,633 +184,127 @@ impl ImageCatalog { continue; } - let mut catalog_manager = - CatalogManager::new(catalog_parts_dir, publisher).map_err(|e| { - CatalogError::Repository(crate::repository::RepositoryError::Other(format!( - "Failed to create catalog manager: {}", - e - ))) - })?; - - // Get all catalog parts - trace!("Getting catalog parts for publisher: {}", publisher); - let parts = catalog_manager.attrs().parts.clone(); - trace!("Catalog parts: {:?}", parts.keys().collect::>()); - - // Load all catalog parts - for part_name in parts.keys() { - trace!("Loading catalog part: {}", part_name); - catalog_manager.load_part(part_name).map_err(|e| { - CatalogError::Repository(crate::repository::RepositoryError::Other(format!( - "Failed to load catalog part: {}", - e - ))) - })?; - } - - // New approach: Merge information across all catalog parts per stem@version, then process once - let mut loaded_parts: Vec<&CatalogPart> = Vec::new(); - for part_name in parts.keys() { - if let Some(part) = catalog_manager.get_part(part_name) { - loaded_parts.push(part); - } - } - self.process_publisher_merged( - &mut catalog_table, - &mut obsoleted_table, + // Build shards using the new sqlite_catalog module + crate::repository::sqlite_catalog::build_shards( + &catalog_parts_dir, publisher, - &loaded_parts, - )?; + shard_dir, + ).map_err(|e| { + CatalogError::Database(format!("Failed to build catalog shards: {}", e.message)) + })?; } - // Drop the tables to release the borrow on tx - drop(catalog_table); - drop(obsoleted_table); - - // Commit the transactions - tx_cat.commit().map_err(|e| { - CatalogError::Database(format!("Failed to commit catalog transaction: {}", e)) - })?; - tx_obs.commit().map_err(|e| { - CatalogError::Database(format!("Failed to commit obsoleted transaction: {}", e)) - })?; - - info!("Catalog built successfully"); + info!("Catalog shards built successfully"); Ok(()) } - /// Process a catalog part and add its packages to the catalog - #[allow(dead_code)] - fn process_catalog_part( - &self, - catalog_table: &mut redb::Table<&str, &[u8]>, - obsoleted_table: &mut redb::Table<&str, &[u8]>, - part_name: &str, - part: &CatalogPart, - publisher: &str, - ) -> Result<()> { - trace!("Processing catalog part for publisher: {}", publisher); + // Removed: process_catalog_part - no longer needed, use build_shards() instead + // + // Removed: process_publisher_merged - no longer needed, use build_shards() instead + // + // Removed: create_or_update_manifest - no longer needed, use build_shards() instead + // + // Removed: ensure_fmri_attribute - no longer needed, use build_shards() instead - // Get packages for this publisher - if let Some(publisher_packages) = part.packages.get(publisher) { - let total_versions: usize = publisher_packages.values().map(|v| v.len()).sum(); - let mut processed: usize = 0; - // Count of packages marked obsolete in this part, including those skipped because they were already marked obsolete in earlier parts. - let mut obsolete_count_incl_skipped: usize = 0; - let mut skipped_obsolete: usize = 0; - let progress_step: usize = 500; // report every N packages - - trace!( - "Found {} package stems ({} versions) for publisher {}", - publisher_packages.len(), - total_versions, - publisher - ); - - // Process each package stem - for (stem, versions) in publisher_packages { - trace!( - "Processing package stem: {} ({} versions)", - stem, - versions.len() - ); - - // Process each package version - for version_entry in versions { - trace!( - "Processing version: {} | actions: {:?}", - version_entry.version, version_entry.actions - ); - - // Create the FMRI - let version = if !version_entry.version.is_empty() { - match crate::fmri::Version::parse(&version_entry.version) { - Ok(v) => Some(v), - Err(e) => { - warn!("Failed to parse version '{}': {}", version_entry.version, e); - continue; - } - } - } else { - None - }; - - let fmri = Fmri::with_publisher(publisher, stem, version); - let catalog_key = format!("{}@{}", stem, version_entry.version); - let obsoleted_key = fmri.to_string(); - - // If this is not the base part and this package/version was already marked - // obsolete in an earlier part (present in obsoleted_table) and is NOT present - // in the catalog_table, skip importing it from this part. - if !part_name.contains(".base") { - let has_catalog = - matches!(catalog_table.get(catalog_key.as_str()), Ok(Some(_))); - if !has_catalog { - let was_obsoleted = - matches!(obsoleted_table.get(obsoleted_key.as_str()), Ok(Some(_))); - if was_obsoleted { - // Count as obsolete for progress accounting, even though we skip processing - obsolete_count_incl_skipped += 1; - skipped_obsolete += 1; - trace!( - "Skipping {} from part {} because it is marked obsolete and not present in catalog", - obsoleted_key, part_name - ); - continue; - } - } - } - - // Check if we already have this package in the catalog - let existing_manifest = match catalog_table.get(catalog_key.as_str()) { - Ok(Some(bytes)) => Some(decode_manifest_bytes(bytes.value())?), - _ => None, - }; - - // Create or update the manifest - let manifest = self.create_or_update_manifest( - existing_manifest, - version_entry, - stem, - publisher, - )?; - - // Check if the package is obsolete - let is_obsolete = self.is_package_obsolete(&manifest); - if is_obsolete { - obsolete_count_incl_skipped += 1; - } - - // Serialize the manifest - let manifest_bytes = serde_json::to_vec(&manifest)?; - - // Store the package in the appropriate table - if is_obsolete { - // Store obsolete packages in the obsoleted table with the full FMRI as key - let empty_bytes: &[u8] = &[0u8; 0]; - obsoleted_table - .insert(obsoleted_key.as_str(), empty_bytes) - .map_err(|e| { - CatalogError::Database(format!( - "Failed to insert into obsoleted table: {}", - e - )) - })?; - } else { - // Store non-obsolete packages in the catalog table with stem@version as a key - let compressed = compress_json_lz4(&manifest_bytes)?; - catalog_table - .insert(catalog_key.as_str(), compressed.as_slice()) - .map_err(|e| { - CatalogError::Database(format!( - "Failed to insert into catalog table: {}", - e - )) - })?; - } - - processed += 1; - if processed % progress_step == 0 { - info!( - "Import progress (publisher {}, part {}): {}/{} versions processed ({} obsolete incl. skipped, {} skipped)", - publisher, - part_name, - processed, - total_versions, - obsolete_count_incl_skipped, - skipped_obsolete - ); - } - } - } - - // Final summary for this part/publisher - info!( - "Finished import for publisher {}, part {}: {} versions processed ({} obsolete incl. skipped, {} skipped)", - publisher, part_name, processed, obsolete_count_incl_skipped, skipped_obsolete - ); - } else { - trace!("No packages found for publisher: {}", publisher); - } - - Ok(()) - } - - /// Process all catalog parts by merging entries per stem@version and deciding once per package - fn process_publisher_merged( - &self, - catalog_table: &mut redb::Table<&str, &[u8]>, - obsoleted_table: &mut redb::Table<&str, &[u8]>, - publisher: &str, - parts: &[&CatalogPart], - ) -> Result<()> { - trace!("Processing merged catalog for publisher: {}", publisher); - - // Build merged map: stem -> version -> PackageVersionEntry (with merged actions/signature) - let mut merged: HashMap> = HashMap::new(); - - for part in parts { - if let Some(publisher_packages) = part.packages.get(publisher) { - for (stem, versions) in publisher_packages { - let stem_map = merged.entry(stem.clone()).or_default(); - for v in versions { - let entry = - stem_map - .entry(v.version.clone()) - .or_insert(PackageVersionEntry { - version: v.version.clone(), - actions: None, - signature_sha1: None, - }); - // Merge signature if not yet set - if entry.signature_sha1.is_none() { - if let Some(sig) = &v.signature_sha1 { - entry.signature_sha1 = Some(sig.clone()); - } - } - // Merge actions, de-duplicating - if let Some(actions) = &v.actions { - let ea = entry.actions.get_or_insert_with(Vec::new); - for a in actions { - if !ea.contains(a) { - ea.push(a.clone()); - } - } - } - } - } - } - } - - // Compute totals for progress logging - let total_versions: usize = merged.values().map(|m| m.len()).sum(); - let mut processed: usize = 0; - let mut obsolete_count: usize = 0; - let progress_step: usize = 500; - - // Deterministic order: sort stems and versions - let mut stems: Vec<&String> = merged.keys().collect(); - stems.sort(); - - for stem in stems { - if let Some(versions_map) = merged.get(stem) { - let mut versions: Vec<&String> = versions_map.keys().collect(); - versions.sort(); - - for ver in versions { - let entry = versions_map.get(ver).expect("version entry exists"); - - // Keys - let catalog_key = format!("{}@{}", stem, entry.version); - - // Read existing manifest if present - let existing_manifest = match catalog_table.get(catalog_key.as_str()) { - Ok(Some(bytes)) => Some(decode_manifest_bytes(bytes.value())?), - _ => None, - }; - - // Build/update manifest with merged actions - let manifest = - self.create_or_update_manifest(existing_manifest, entry, stem, publisher)?; - - // Obsolete decision based on merged actions in manifest - let is_obsolete = self.is_package_obsolete(&manifest); - if is_obsolete { - obsolete_count += 1; - } - - // Serialize and write - if is_obsolete { - // Compute full FMRI for obsoleted key - let version_obj = if !entry.version.is_empty() { - match crate::fmri::Version::parse(&entry.version) { - Ok(v) => Some(v), - Err(_) => None, - } - } else { - None - }; - let fmri = Fmri::with_publisher(publisher, stem, version_obj); - let obsoleted_key = fmri.to_string(); - let empty_bytes: &[u8] = &[0u8; 0]; - obsoleted_table - .insert(obsoleted_key.as_str(), empty_bytes) - .map_err(|e| { - CatalogError::Database(format!( - "Failed to insert into obsoleted table: {}", - e - )) - })?; - } else { - let manifest_bytes = serde_json::to_vec(&manifest)?; - let compressed = compress_json_lz4(&manifest_bytes)?; - catalog_table - .insert(catalog_key.as_str(), compressed.as_slice()) - .map_err(|e| { - CatalogError::Database(format!( - "Failed to insert into catalog table: {}", - e - )) - })?; - } - - processed += 1; - if processed % progress_step == 0 { - info!( - "Import progress (publisher {}, merged): {}/{} versions processed ({} obsolete)", - publisher, processed, total_versions, obsolete_count - ); - } - } - } - } - - info!( - "Finished merged import for publisher {}: {} versions processed ({} obsolete)", - publisher, processed, obsolete_count - ); - - Ok(()) - } - - /// Create or update a manifest from a package version entry - fn create_or_update_manifest( - &self, - existing_manifest: Option, - version_entry: &PackageVersionEntry, - stem: &str, - publisher: &str, - ) -> Result { - // Start with the existing manifest or create a new one - let mut manifest = existing_manifest.unwrap_or_else(Manifest::new); - - // Parse and add actions from the version entry - if let Some(actions) = &version_entry.actions { - for action_str in actions { - // Parse each action string to extract attributes we care about in the catalog - if action_str.starts_with("set ") { - // Format is typically "set name=pkg.key value=value" - if let Some(name_part) = action_str.split_whitespace().nth(1) { - if name_part.starts_with("name=") { - // Extract the key (after "name=") - let key = &name_part[5..]; - - // Extract the value (after "value=") - if let Some(value_part) = action_str.split_whitespace().nth(2) { - if value_part.starts_with("value=") { - let mut value = &value_part[6..]; - - // Remove quotes if present - if value.starts_with('"') && value.ends_with('"') { - value = &value[1..value.len() - 1]; - } - - // Add or update the attribute in the manifest - let attr_index = - manifest.attributes.iter().position(|attr| attr.key == key); - if let Some(index) = attr_index { - manifest.attributes[index].values = vec![value.to_string()]; - } else { - let mut attr = crate::actions::Attr::default(); - attr.key = key.to_string(); - attr.values = vec![value.to_string()]; - manifest.attributes.push(attr); - } - } - } - } - } - } else if action_str.starts_with("depend ") { - // Example: "depend fmri=desktop/mate/caja type=require" - let rest = &action_str[7..]; // strip leading "depend " - let mut dep_type: String = String::new(); - let mut dep_predicate: Option = None; - let mut dep_fmris: Vec = Vec::new(); - let mut root_image: String = String::new(); - - for tok in rest.split_whitespace() { - if let Some((k, v)) = tok.split_once('=') { - match k { - "type" => dep_type = v.to_string(), - "predicate" => { - if let Ok(f) = crate::fmri::Fmri::parse(v) { - dep_predicate = Some(f); - } - } - "fmri" => { - if let Ok(f) = crate::fmri::Fmri::parse(v) { - dep_fmris.push(f); - } - } - "root-image" => { - root_image = v.to_string(); - } - _ => { /* ignore other props for catalog */ } - } - } - } - - // For each fmri property, add a Dependency entry - for f in dep_fmris { - let mut d = crate::actions::Dependency::default(); - d.fmri = Some(f); - d.dependency_type = dep_type.clone(); - d.predicate = dep_predicate.clone(); - d.root_image = root_image.clone(); - manifest.dependencies.push(d); - } - } - } - } - - // Ensure the manifest has the correct FMRI attribute - // Create a Version object from the version string - let version = if !version_entry.version.is_empty() { - match crate::fmri::Version::parse(&version_entry.version) { - Ok(v) => Some(v), - Err(e) => { - // Map the FmriError to a CatalogError - return Err(CatalogError::Repository( - crate::repository::RepositoryError::Other(format!( - "Invalid version format: {}", - e - )), - )); - } - } - } else { - None - }; - - // Create the FMRI with publisher, stem, and version - let fmri = Fmri::with_publisher(publisher, stem, version); - self.ensure_fmri_attribute(&mut manifest, &fmri); - - Ok(manifest) - } - - /// Ensure the manifest has the correct FMRI attribute - fn ensure_fmri_attribute(&self, manifest: &mut Manifest, fmri: &Fmri) { - // Check if the manifest already has an FMRI attribute - let has_fmri = manifest - .attributes - .iter() - .any(|attr| attr.key == "pkg.fmri"); - - // If not, add it - if !has_fmri { - let mut attr = crate::actions::Attr::default(); - attr.key = "pkg.fmri".to_string(); - attr.values = vec![fmri.to_string()]; - manifest.attributes.push(attr); - } - } - - /// Check if a package is obsolete + /// Check if a package is obsolete (deprecated - use free function is_package_obsolete instead) fn is_package_obsolete(&self, manifest: &Manifest) -> bool { - manifest.attributes.iter().any(|attr| { - attr.key == "pkg.obsolete" && attr.values.first().map_or(false, |v| v == "true") - }) + is_package_obsolete(manifest) } /// Query the catalog for packages matching a pattern + /// + /// Reads from active.db and obsolete.db SQLite shards. pub fn query_packages(&self, pattern: Option<&str>) -> Result> { - // Open the catalog database - let db_cat = Database::open(&self.db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open catalog database: {}", e)) - })?; - // Begin a read transaction - let tx_cat = db_cat - .begin_read() - .map_err(|e| CatalogError::Database(format!("Failed to begin transaction: {}", e)))?; - - // Open the catalog table - let catalog_table = tx_cat - .open_table(CATALOG_TABLE) - .map_err(|e| CatalogError::Database(format!("Failed to open catalog table: {}", e)))?; - - // Open the obsoleted database - let db_obs = Database::open(&self.obsoleted_db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open obsoleted database: {}", e)) - })?; - let tx_obs = db_obs - .begin_read() - .map_err(|e| CatalogError::Database(format!("Failed to begin transaction: {}", e)))?; - let obsoleted_table = tx_obs.open_table(OBSOLETED_TABLE).map_err(|e| { - CatalogError::Database(format!("Failed to open obsoleted table: {}", e)) - })?; + use rusqlite::{Connection, OpenFlags}; let mut results = Vec::new(); - // Process the catalog table (non-obsolete packages) - // Iterate through all entries in the table - for entry_result in catalog_table.iter().map_err(|e| { - CatalogError::Database(format!("Failed to iterate catalog table: {}", e)) - })? { - let (key, value) = entry_result.map_err(|e| { - CatalogError::Database(format!("Failed to get entry from catalog table: {}", e)) - })?; - let key_str = key.value(); + // Query active.db for non-obsolete packages + if self.db_path.exists() { + let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY) + .map_err(|e| CatalogError::Database(format!("Failed to open active.db: {}", e)))?; - // Skip if the key doesn't match the pattern - if let Some(pattern) = pattern { - if !key_str.contains(pattern) { - continue; - } - } + let mut stmt = conn + .prepare("SELECT stem, version, publisher FROM packages") + .map_err(|e| CatalogError::Database(format!("Failed to prepare query: {}", e)))?; - // Parse the key to get stem and version - let parts: Vec<&str> = key_str.split('@').collect(); - if parts.len() != 2 { - warn!("Invalid key format: {}", key_str); - continue; - } - - let stem = parts[0]; - let version = parts[1]; - - // Deserialize the manifest - let manifest: Manifest = decode_manifest_bytes(value.value())?; - - // Extract the publisher from the FMRI attribute - let publisher = manifest - .attributes - .iter() - .find(|attr| attr.key == "pkg.fmri") - .map(|attr| { - if let Some(fmri_str) = attr.values.first() { - // Parse the FMRI string - match Fmri::parse(fmri_str) { - Ok(fmri) => fmri.publisher.unwrap_or_else(|| "unknown".to_string()), - Err(_) => "unknown".to_string(), - } - } else { - "unknown".to_string() - } + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) }) - .unwrap_or_else(|| "unknown".to_string()); + .map_err(|e| CatalogError::Database(format!("Failed to query packages: {}", e)))?; - // Create a Version object from the version string - let version_obj = if !version.is_empty() { - match crate::fmri::Version::parse(version) { - Ok(v) => Some(v), - Err(_) => None, + for row_result in rows { + let (stem, version, publisher) = row_result + .map_err(|e| CatalogError::Database(format!("Failed to read row: {}", e)))?; + + // Apply pattern filter + if let Some(pat) = pattern { + if !stem.contains(pat) && !version.contains(pat) { + continue; + } } - } else { - None - }; - // Create the FMRI with publisher, stem, and version - let fmri = Fmri::with_publisher(&publisher, stem, version_obj); + // Parse version + let version_obj = crate::fmri::Version::parse(&version).ok(); + let fmri = Fmri::with_publisher(&publisher, &stem, version_obj); - // Add to results (non-obsolete) - results.push(PackageInfo { - fmri, - obsolete: false, - publisher, - }); + results.push(PackageInfo { + fmri, + obsolete: false, + publisher, + }); + } } - // Process the obsoleted table (obsolete packages) - // Iterate through all entries in the table - for entry_result in obsoleted_table.iter().map_err(|e| { - CatalogError::Database(format!("Failed to iterate obsoleted table: {}", e)) - })? { - let (key, _) = entry_result.map_err(|e| { - CatalogError::Database(format!("Failed to get entry from obsoleted table: {}", e)) - })?; - let key_str = key.value(); + // Query obsolete.db for obsolete packages + if self.obsoleted_db_path.exists() { + let conn = Connection::open_with_flags( + &self.obsoleted_db_path, + OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|e| CatalogError::Database(format!("Failed to open obsolete.db: {}", e)))?; - // Skip if the key doesn't match the pattern - if let Some(pattern) = pattern { - if !key_str.contains(pattern) { - continue; - } - } + let mut stmt = conn + .prepare("SELECT publisher, stem, version FROM obsolete_packages") + .map_err(|e| CatalogError::Database(format!("Failed to prepare query: {}", e)))?; - // Parse the key to get the FMRI - match Fmri::parse(key_str) { - Ok(fmri) => { - // Extract the publisher - let publisher = fmri - .publisher - .clone() - .unwrap_or_else(|| "unknown".to_string()); + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }) + .map_err(|e| { + CatalogError::Database(format!("Failed to query obsolete packages: {}", e)) + })?; - // Add to results (obsolete) - results.push(PackageInfo { - fmri, - obsolete: true, - publisher, - }); - } - Err(e) => { - warn!( - "Failed to parse FMRI from obsoleted table key: {}: {}", - key_str, e - ); - continue; + for row_result in rows { + let (publisher, stem, version) = row_result + .map_err(|e| CatalogError::Database(format!("Failed to read row: {}", e)))?; + + // Apply pattern filter + if let Some(pat) = pattern { + if !stem.contains(pat) && !version.contains(pat) { + continue; + } } + + // Parse version + let version_obj = crate::fmri::Version::parse(&version).ok(); + let fmri = Fmri::with_publisher(&publisher, &stem, version_obj); + + results.push(PackageInfo { + fmri, + obsolete: true, + publisher, + }); } } @@ -1139,52 +312,126 @@ impl ImageCatalog { } /// Get a manifest from the catalog + /// + /// Note: The SQLite shards don't store manifests. This method checks if the package + /// exists in active.db or obsolete.db and returns a minimal manifest if found. + /// For full manifests, use get_manifest_from_repository or the local manifest cache. pub fn get_manifest(&self, fmri: &Fmri) -> Result> { - // Open the catalog database - let db_cat = Database::open(&self.db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open catalog database: {}", e)) - })?; - // Begin a read transaction - let tx_cat = db_cat - .begin_read() - .map_err(|e| CatalogError::Database(format!("Failed to begin transaction: {}", e)))?; + use rusqlite::{Connection, OpenFlags}; - // Open the catalog table - let catalog_table = tx_cat - .open_table(CATALOG_TABLE) - .map_err(|e| CatalogError::Database(format!("Failed to open catalog table: {}", e)))?; + // Check active.db + if self.db_path.exists() { + let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY) + .map_err(|e| CatalogError::Database(format!("Failed to open active.db: {}", e)))?; - // Create the key for the catalog table (stem@version) - let catalog_key = format!("{}@{}", fmri.stem(), fmri.version()); + let count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM packages WHERE stem = ?1 AND version = ?2", + rusqlite::params![fmri.stem(), fmri.version()], + |row| row.get(0), + ) + .unwrap_or(0); - // Try to get the manifest from the catalog table - if let Ok(Some(bytes)) = catalog_table.get(catalog_key.as_str()) { - return Ok(Some(decode_manifest_bytes(bytes.value())?)); + if count > 0 { + // Package exists but we don't have the full manifest in the shard + // Return a minimal manifest with just the FMRI + let mut manifest = Manifest::new(); + let mut attr = crate::actions::Attr::default(); + attr.key = "pkg.fmri".to_string(); + attr.values = vec![fmri.to_string()]; + manifest.attributes.push(attr); + return Ok(Some(manifest)); + } } - // If not found in catalog DB, check obsoleted DB - let db_obs = Database::open(&self.obsoleted_db_path).map_err(|e| { - CatalogError::Database(format!("Failed to open obsoleted database: {}", e)) - })?; - let tx_obs = db_obs - .begin_read() - .map_err(|e| CatalogError::Database(format!("Failed to begin transaction: {}", e)))?; - let obsoleted_table = tx_obs.open_table(OBSOLETED_TABLE).map_err(|e| { - CatalogError::Database(format!("Failed to open obsoleted table: {}", e)) - })?; - let obsoleted_key = fmri.to_string(); - if let Ok(Some(_)) = obsoleted_table.get(obsoleted_key.as_str()) { - let mut manifest = Manifest::new(); - let mut attr = crate::actions::Attr::default(); - attr.key = "pkg.fmri".to_string(); - attr.values = vec![fmri.to_string()]; - manifest.attributes.push(attr); - let mut attr = crate::actions::Attr::default(); - attr.key = "pkg.obsolete".to_string(); - attr.values = vec!["true".to_string()]; - manifest.attributes.push(attr); - return Ok(Some(manifest)); + // Check obsolete.db + if self.obsoleted_db_path.exists() { + let conn = Connection::open_with_flags( + &self.obsoleted_db_path, + OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|e| CatalogError::Database(format!("Failed to open obsolete.db: {}", e)))?; + + let count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM obsolete_packages WHERE stem = ?1 AND version = ?2", + rusqlite::params![fmri.stem(), fmri.version()], + |row| row.get(0), + ) + .unwrap_or(0); + + if count > 0 { + // Create a minimal obsolete manifest + let mut manifest = Manifest::new(); + let mut attr = crate::actions::Attr::default(); + attr.key = "pkg.fmri".to_string(); + attr.values = vec![fmri.to_string()]; + manifest.attributes.push(attr); + + let mut attr = crate::actions::Attr::default(); + attr.key = "pkg.obsolete".to_string(); + attr.values = vec!["true".to_string()]; + manifest.attributes.push(attr); + + return Ok(Some(manifest)); + } } + Ok(None) } + + /// Dump the contents of a specific table to stdout for debugging + /// + /// Deprecated: This method used redb. Needs reimplementation for SQLite. + pub fn dump_table(&self, table_name: &str) -> Result<()> { + Err(CatalogError::Database(format!( + "dump_table is not yet implemented for SQLite catalog (requested table: {})", + table_name + ))) + } + + /// Dump the contents of all tables to stdout for debugging + /// + /// Deprecated: This method used redb. Needs reimplementation for SQLite. + pub fn dump_all_tables(&self) -> Result<()> { + Err(CatalogError::Database( + "dump_all_tables is not yet implemented for SQLite catalog".to_string(), + )) + } + + /// Dump the contents of the catalog table (private helper) + fn dump_catalog_table(&self, _tx: &()) -> Result<()> { + Err(CatalogError::Database( + "dump_catalog_table is not yet implemented for SQLite catalog".to_string(), + )) + } + + /// Dump the contents of the obsoleted table (private helper) + fn dump_obsoleted_table(&self, _tx: &()) -> Result<()> { + Err(CatalogError::Database( + "dump_obsoleted_table is not yet implemented for SQLite catalog".to_string(), + )) + } + + /// Get database statistics + /// + /// Deprecated: This method used redb. Needs reimplementation for SQLite. + pub fn get_db_stats(&self) -> Result<()> { + Err(CatalogError::Database( + "get_db_stats is not yet implemented for SQLite catalog".to_string(), + )) + } +} + +// Removed all the implementation code for the following methods since they're no longer used: +// - process_catalog_part +// - process_publisher_merged +// - create_or_update_manifest +// - ensure_fmri_attribute + +/// Helper function to parse an action string and extract the key-value pairs +#[allow(dead_code)] +fn parse_action(_action_str: &str) -> HashMap { + // This is legacy code for the old catalog building. It's not needed anymore. + HashMap::new() } diff --git a/libips/src/image/installed.rs b/libips/src/image/installed.rs index 8bde629..cdec17e 100644 --- a/libips/src/image/installed.rs +++ b/libips/src/image/installed.rs @@ -1,7 +1,8 @@ use crate::actions::Manifest; use crate::fmri::Fmri; +use crate::repository::sqlite_catalog::INSTALLED_SCHEMA; use miette::Diagnostic; -use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition}; +use rusqlite::{Connection, OpenFlags}; use serde::{Deserialize, Serialize}; use std::fs; use std::path::{Path, PathBuf}; @@ -9,11 +10,6 @@ use std::str::FromStr; use thiserror::Error; use tracing::info; -/// Table definition for the installed packages database -/// Key: full FMRI including publisher (pkg://publisher/stem@version) -/// Value: serialized Manifest -pub const INSTALLED_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("installed"); - /// Errors that can occur when working with the installed packages database #[derive(Error, Debug, Diagnostic)] pub enum InstalledError { @@ -38,6 +34,12 @@ pub enum InstalledError { PackageNotFound(String), } +impl From for InstalledError { + fn from(e: rusqlite::Error) -> Self { + InstalledError::Database(format!("SQLite error: {}", e)) + } +} + /// Result type for installed packages operations pub type Result = std::result::Result; @@ -58,17 +60,6 @@ pub struct InstalledPackages { } impl InstalledPackages { - // Note on borrowing and redb: - // When using redb, there's a potential borrowing issue when working with transactions and tables. - // The issue occurs because: - // 1. Tables borrow from the transaction they were opened from - // 2. When committing a transaction with tx.commit(), the transaction is moved - // 3. If a table is still borrowing from the transaction when commit() is called, Rust's borrow checker will prevent the move - // - // To fix this issue, we use block scopes {} around table operations to ensure that the table - // objects are dropped (and their borrows released) before committing the transaction. - // This pattern is used in all methods that commit transactions after table operations. - /// Create a new installed packages database pub fn new>(db_path: P) -> Self { InstalledPackages { @@ -78,98 +69,41 @@ impl InstalledPackages { /// Dump the contents of the installed table to stdout for debugging pub fn dump_installed_table(&self) -> Result<()> { - // Open the database - let db = Database::open(&self.db_path) - .map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?; + let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; - // Begin a read transaction - let tx = db - .begin_read() - .map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?; + let mut stmt = conn.prepare("SELECT fmri, manifest FROM installed")?; + let mut rows = stmt.query([])?; - // Open the installed table - match tx.open_table(INSTALLED_TABLE) { - Ok(table) => { - let mut count = 0; - for entry_result in table.iter().map_err(|e| { - InstalledError::Database(format!("Failed to iterate installed table: {}", e)) - })? { - let (key, value) = entry_result.map_err(|e| { - InstalledError::Database(format!( - "Failed to get entry from installed table: {}", - e - )) - })?; - let key_str = key.value(); + let mut count = 0; + while let Some(row) = rows.next()? { + let fmri_str: String = row.get(0)?; + let manifest_bytes: Vec = row.get(1)?; - // Try to deserialize the manifest - match serde_json::from_slice::(value.value()) { - Ok(manifest) => { - // Extract the publisher from the FMRI attribute - let publisher = manifest - .attributes - .iter() - .find(|attr| attr.key == "pkg.fmri") - .and_then(|attr| attr.values.first().cloned()) - .unwrap_or_else(|| "unknown".to_string()); - - println!("Key: {}", key_str); - println!(" FMRI: {}", publisher); - println!(" Attributes: {}", manifest.attributes.len()); - println!(" Files: {}", manifest.files.len()); - println!(" Directories: {}", manifest.directories.len()); - println!(" Dependencies: {}", manifest.dependencies.len()); - } - Err(e) => { - println!("Key: {}", key_str); - println!(" Error deserializing manifest: {}", e); - } - } - count += 1; + match serde_json::from_slice::(&manifest_bytes) { + Ok(manifest) => { + println!("FMRI: {}", fmri_str); + println!(" Attributes: {}", manifest.attributes.len()); + println!(" Files: {}", manifest.files.len()); + println!(" Directories: {}", manifest.directories.len()); + println!(" Dependencies: {}", manifest.dependencies.len()); + } + Err(e) => { + println!("FMRI: {}", fmri_str); + println!(" Error deserializing manifest: {}", e); } - println!("Total entries in installed table: {}", count); - Ok(()) - } - Err(e) => { - println!("Error opening installed table: {}", e); - Err(InstalledError::Database(format!( - "Failed to open installed table: {}", - e - ))) } + count += 1; } + println!("Total entries in installed table: {}", count); + Ok(()) } /// Get database statistics pub fn get_db_stats(&self) -> Result<()> { - // Open the database - let db = Database::open(&self.db_path) - .map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?; + let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; - // Begin a read transaction - let tx = db - .begin_read() - .map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?; + let installed_count: i64 = conn.query_row("SELECT COUNT(*) FROM installed", [], |row| row.get(0))?; - // Get table statistics - let mut installed_count = 0; - - // Count installed entries - if let Ok(table) = tx.open_table(INSTALLED_TABLE) { - for result in table.iter().map_err(|e| { - InstalledError::Database(format!("Failed to iterate installed table: {}", e)) - })? { - let _ = result.map_err(|e| { - InstalledError::Database(format!( - "Failed to get entry from installed table: {}", - e - )) - })?; - installed_count += 1; - } - } - - // Print statistics println!("Database path: {}", self.db_path.display()); println!("Table statistics:"); println!(" Installed table: {} entries", installed_count); @@ -180,72 +114,33 @@ impl InstalledPackages { /// Initialize the installed packages database pub fn init_db(&self) -> Result<()> { - // Create a parent directory if it doesn't exist + // Create parent directory if it doesn't exist if let Some(parent) = self.db_path.parent() { fs::create_dir_all(parent)?; } - // Open or create the database - let db = Database::create(&self.db_path) - .map_err(|e| InstalledError::Database(format!("Failed to create database: {}", e)))?; + // Create or open the database + let conn = Connection::open(&self.db_path)?; - // Create tables - let tx = db - .begin_write() - .map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?; - - tx.open_table(INSTALLED_TABLE).map_err(|e| { - InstalledError::Database(format!("Failed to create installed table: {}", e)) - })?; - - tx.commit().map_err(|e| { - InstalledError::Database(format!("Failed to commit transaction: {}", e)) - })?; + // Execute schema + conn.execute_batch(INSTALLED_SCHEMA)?; Ok(()) } /// Add a package to the installed packages database pub fn add_package(&self, fmri: &Fmri, manifest: &Manifest) -> Result<()> { - // Open the database - let db = Database::open(&self.db_path) - .map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?; + let mut conn = Connection::open(&self.db_path)?; - // Begin a writing transaction - let tx = db - .begin_write() - .map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?; - - // Create the key (full FMRI including publisher) let key = fmri.to_string(); - - // Serialize the manifest let manifest_bytes = serde_json::to_vec(manifest)?; - // Use a block scope to ensure the table is dropped before committing the transaction - { - // Open the installed table - let mut installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| { - InstalledError::Database(format!("Failed to open installed table: {}", e)) - })?; - - // Insert the package into the installed table - installed_table - .insert(key.as_str(), manifest_bytes.as_slice()) - .map_err(|e| { - InstalledError::Database(format!( - "Failed to insert into installed table: {}", - e - )) - })?; - - // The table is dropped at the end of this block, releasing its borrow of tx - } - - // Commit the transaction - tx.commit().map_err(|e| { - InstalledError::Database(format!("Failed to commit transaction: {}", e)) - })?; + let tx = conn.transaction()?; + tx.execute( + "INSERT OR REPLACE INTO installed (fmri, manifest) VALUES (?1, ?2)", + rusqlite::params![key, manifest_bytes], + )?; + tx.commit()?; info!("Added package to installed database: {}", key); Ok(()) @@ -253,42 +148,24 @@ impl InstalledPackages { /// Remove a package from the installed packages database pub fn remove_package(&self, fmri: &Fmri) -> Result<()> { - // Open the database - let db = Database::open(&self.db_path) - .map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?; + let mut conn = Connection::open(&self.db_path)?; - // Begin a writing transaction - let tx = db - .begin_write() - .map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?; - - // Create the key (full FMRI including publisher) let key = fmri.to_string(); - // Use a block scope to ensure the table is dropped before committing the transaction - { - // Open the installed table - let mut installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| { - InstalledError::Database(format!("Failed to open installed table: {}", e)) - })?; + // Check if the package exists + let exists: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM installed WHERE fmri = ?1)", + rusqlite::params![key], + |row| row.get(0), + )?; - // Check if the package exists - if let Ok(None) = installed_table.get(key.as_str()) { - return Err(InstalledError::PackageNotFound(key)); - } - - // Remove the package from the installed table - installed_table.remove(key.as_str()).map_err(|e| { - InstalledError::Database(format!("Failed to remove from installed table: {}", e)) - })?; - - // The table is dropped at the end of this block, releasing its borrow of tx + if !exists { + return Err(InstalledError::PackageNotFound(key)); } - // Commit the transaction - tx.commit().map_err(|e| { - InstalledError::Database(format!("Failed to commit transaction: {}", e)) - })?; + let tx = conn.transaction()?; + tx.execute("DELETE FROM installed WHERE fmri = ?1", rusqlite::params![key])?; + tx.commit()?; info!("Removed package from installed database: {}", key); Ok(()) @@ -296,127 +173,60 @@ impl InstalledPackages { /// Query the installed packages database for packages matching a pattern pub fn query_packages(&self, pattern: Option<&str>) -> Result> { - // Open the database - let db = Database::open(&self.db_path) - .map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?; + let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; - // Begin a read transaction - let tx = db - .begin_read() - .map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?; - - // Use a block scope to ensure the table is dropped when no longer needed - let results = { - // Open the installed table - let installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| { - InstalledError::Database(format!("Failed to open installed table: {}", e)) - })?; - - let mut results = Vec::new(); - - // Process the installed table - // Iterate through all entries in the table - for entry_result in installed_table.iter().map_err(|e| { - InstalledError::Database(format!("Failed to iterate installed table: {}", e)) - })? { - let (key, _) = entry_result.map_err(|e| { - InstalledError::Database(format!( - "Failed to get entry from installed table: {}", - e - )) - })?; - let key_str = key.value(); - - // Skip if the key doesn't match the pattern - if let Some(pattern) = pattern { - if !key_str.contains(pattern) { - continue; - } - } - - // Parse the key to get the FMRI - let fmri = Fmri::from_str(key_str)?; - - // Get the publisher (handling the Option) - let publisher = fmri - .publisher - .clone() - .unwrap_or_else(|| "unknown".to_string()); - - // Add to results - results.push(InstalledPackageInfo { fmri, publisher }); - } - - results - // The table is dropped at the end of this block + let query = if let Some(pattern) = pattern { + format!("SELECT fmri FROM installed WHERE fmri LIKE '%{}%'", pattern.replace('\'', "''")) + } else { + "SELECT fmri FROM installed".to_string() }; + let mut stmt = conn.prepare(&query)?; + let mut rows = stmt.query([])?; + + let mut results = Vec::new(); + while let Some(row) = rows.next()? { + let fmri_str: String = row.get(0)?; + let fmri = Fmri::from_str(&fmri_str)?; + let publisher = fmri.publisher.clone().unwrap_or_else(|| "unknown".to_string()); + results.push(InstalledPackageInfo { fmri, publisher }); + } + Ok(results) } /// Get a manifest from the installed packages database pub fn get_manifest(&self, fmri: &Fmri) -> Result> { - // Open the database - let db = Database::open(&self.db_path) - .map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?; + let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; - // Begin a read transaction - let tx = db - .begin_read() - .map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?; - - // Create the key (full FMRI including publisher) let key = fmri.to_string(); + let result = conn.query_row( + "SELECT manifest FROM installed WHERE fmri = ?1", + rusqlite::params![key], + |row| { + let bytes: Vec = row.get(0)?; + Ok(bytes) + }, + ); - // Use a block scope to ensure the table is dropped when no longer needed - let manifest_option = { - // Open the installed table - let installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| { - InstalledError::Database(format!("Failed to open installed table: {}", e)) - })?; - - // Try to get the manifest from the installed table - if let Ok(Some(bytes)) = installed_table.get(key.as_str()) { - Some(serde_json::from_slice(bytes.value())?) - } else { - None - } - // The table is dropped at the end of this block - }; - - Ok(manifest_option) + match result { + Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes)?)), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(e.into()), + } } /// Check if a package is installed pub fn is_installed(&self, fmri: &Fmri) -> Result { - // Open the database - let db = Database::open(&self.db_path) - .map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?; + let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; - // Begin a read transaction - let tx = db - .begin_read() - .map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?; - - // Create the key (full FMRI including publisher) let key = fmri.to_string(); + let exists: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM installed WHERE fmri = ?1)", + rusqlite::params![key], + |row| row.get(0), + )?; - // Use a block scope to ensure the table is dropped when no longer needed - let is_installed = { - // Open the installed table - let installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| { - InstalledError::Database(format!("Failed to open installed table: {}", e)) - })?; - - // Check if the package exists - if let Ok(Some(_)) = installed_table.get(key.as_str()) { - true - } else { - false - } - // The table is dropped at the end of this block - }; - - Ok(is_installed) + Ok(exists) } } diff --git a/libips/src/image/installed_tests.rs b/libips/src/image/installed_tests.rs index 05dd84d..5a6241f 100644 --- a/libips/src/image/installed_tests.rs +++ b/libips/src/image/installed_tests.rs @@ -1,7 +1,7 @@ use super::*; use crate::actions::{Attr, Manifest}; use crate::fmri::Fmri; -use redb::{Database, ReadableTable}; +use rusqlite::Connection; use std::str::FromStr; use tempfile::tempdir; @@ -82,7 +82,7 @@ fn test_installed_packages() { fn test_installed_packages_key_format() { // Create a temporary directory for the test let temp_dir = tempdir().unwrap(); - let db_path = temp_dir.path().join("installed.redb"); + let db_path = temp_dir.path().join("installed.db"); // Create the installed packages database let installed = InstalledPackages::new(&db_path); @@ -104,15 +104,15 @@ fn test_installed_packages_key_format() { installed.add_package(&fmri, &manifest).unwrap(); // Open the database directly to check the key format - let db = Database::open(&db_path).unwrap(); - let tx = db.begin_read().unwrap(); - let table = tx.open_table(installed::INSTALLED_TABLE).unwrap(); + let conn = Connection::open(&db_path).unwrap(); + let mut stmt = conn.prepare("SELECT fmri FROM installed").unwrap(); + let mut rows = stmt.query([]).unwrap(); - // Iterate through the keys + // Collect the keys let mut keys = Vec::new(); - for entry in table.iter().unwrap() { - let (key, _) = entry.unwrap(); - keys.push(key.value().to_string()); + while let Some(row) = rows.next().unwrap() { + let fmri_str: String = row.get(0).unwrap(); + keys.push(fmri_str); } // Verify that there is one key and it has the correct format diff --git a/libips/src/image/mod.rs b/libips/src/image/mod.rs index ce895d7..96f1e42 100644 --- a/libips/src/image/mod.rs +++ b/libips/src/image/mod.rs @@ -4,7 +4,7 @@ mod tests; use miette::Diagnostic; use properties::*; -use redb::{Database, ReadableDatabase, ReadableTable}; +use rusqlite::{Connection, OpenFlags}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fs::{self, File}; @@ -15,7 +15,7 @@ use crate::repository::{FileBackend, ReadableRepository, RepositoryError, RestBa // Export the catalog module pub mod catalog; -use catalog::{INCORPORATE_TABLE, ImageCatalog, PackageInfo}; +use catalog::{ImageCatalog, PackageInfo}; // Export the installed packages module pub mod installed; @@ -79,6 +79,12 @@ pub enum ImageError { NoPublishers, } +impl From for ImageError { + fn from(e: rusqlite::Error) -> Self { + ImageError::Database(format!("SQLite error: {}", e)) + } +} + pub type Result = std::result::Result; /// Type of image, either Full (base path of "/") or Partial (attached to a full image) @@ -281,7 +287,7 @@ impl Image { /// Returns the path to the installed packages database pub fn installed_db_path(&self) -> PathBuf { - self.metadata_dir().join("installed.redb") + self.metadata_dir().join("installed.db") } /// Returns the path to the manifest directory @@ -294,14 +300,31 @@ impl Image { self.metadata_dir().join("catalog") } - /// Returns the path to the catalog database - pub fn catalog_db_path(&self) -> PathBuf { - self.metadata_dir().join("catalog.redb") + /// Returns the path to the active catalog database (packages and dependencies) + pub fn active_db_path(&self) -> PathBuf { + self.metadata_dir().join("active.db") } - /// Returns the path to the obsoleted packages database (separate DB) + /// Returns the path to the obsoleted packages database + pub fn obsolete_db_path(&self) -> PathBuf { + self.metadata_dir().join("obsolete.db") + } + + /// Returns the path to the full-text search database + pub fn fts_db_path(&self) -> PathBuf { + self.metadata_dir().join("fts.db") + } + + /// Deprecated: Use active_db_path() instead + #[deprecated(note = "Use active_db_path() instead")] + pub fn catalog_db_path(&self) -> PathBuf { + self.active_db_path() + } + + /// Deprecated: Use obsolete_db_path() instead + #[deprecated(note = "Use obsolete_db_path() instead")] pub fn obsoleted_db_path(&self) -> PathBuf { - self.metadata_dir().join("obsoleted.redb") + self.obsolete_db_path() } /// Creates the metadata directory if it doesn't exist @@ -528,14 +551,18 @@ impl Image { /// Initialize the catalog database pub fn init_catalog_db(&self) -> Result<()> { - let catalog = ImageCatalog::new( - self.catalog_dir(), - self.catalog_db_path(), - self.obsoleted_db_path(), - ); - catalog.init_db().map_err(|e| { - ImageError::Database(format!("Failed to initialize catalog database: {}", e)) - }) + use crate::repository::sqlite_catalog::ACTIVE_SCHEMA; + + let path = self.active_db_path(); + if let Some(parent) = path.parent() { + fs::create_dir_all(parent)?; + } + + let conn = Connection::open(&path)?; + conn.execute_batch(ACTIVE_SCHEMA) + .map_err(|e| ImageError::Database(format!("Failed to initialize catalog database: {}", e)))?; + + Ok(()) } /// Download catalogs from all configured publishers and build the merged catalog @@ -657,65 +684,102 @@ impl Image { /// Look up an incorporation lock for a given stem. /// Returns Some(release) if a lock exists, otherwise None. pub fn get_incorporated_release(&self, stem: &str) -> Result> { - let db = Database::open(self.catalog_db_path()) - .map_err(|e| ImageError::Database(format!("Failed to open catalog database: {}", e)))?; - let tx = db.begin_read().map_err(|e| { - ImageError::Database(format!("Failed to begin read transaction: {}", e)) - })?; - match tx.open_table(INCORPORATE_TABLE) { - Ok(table) => match table.get(stem) { - Ok(Some(val)) => Ok(Some(String::from_utf8_lossy(val.value()).to_string())), - Ok(None) => Ok(None), - Err(e) => Err(ImageError::Database(format!( - "Failed to read incorporate lock: {}", - e - ))), - }, - Err(_) => Ok(None), + let conn = Connection::open_with_flags( + &self.active_db_path(), + OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|e| ImageError::Database(format!("Failed to open catalog database: {}", e)))?; + + let result = conn.query_row( + "SELECT release FROM incorporate_locks WHERE stem = ?1", + rusqlite::params![stem], + |row| row.get(0), + ); + + match result { + Ok(release) => Ok(Some(release)), + Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None), + Err(e) => Err(ImageError::Database(format!( + "Failed to read incorporate lock: {}", + e + ))), } } /// Add an incorporation lock for a stem to a specific release. - /// Fails if a lock already exists for the stem. + /// Uses INSERT OR REPLACE, so will update if a lock already exists. pub fn add_incorporation_lock(&self, stem: &str, release: &str) -> Result<()> { - let db = Database::open(self.catalog_db_path()) + let mut conn = Connection::open(&self.active_db_path()) .map_err(|e| ImageError::Database(format!("Failed to open catalog database: {}", e)))?; - let tx = db.begin_write().map_err(|e| { + + let tx = conn.transaction().map_err(|e| { ImageError::Database(format!("Failed to begin write transaction: {}", e)) })?; - { - let mut table = tx.open_table(INCORPORATE_TABLE).map_err(|e| { - ImageError::Database(format!("Failed to open incorporate table: {}", e)) - })?; - if let Ok(Some(_)) = table.get(stem) { - return Err(ImageError::Database(format!( - "Incorporation lock already exists for stem {}", - stem - ))); - } - table.insert(stem, release.as_bytes()).map_err(|e| { - ImageError::Database(format!("Failed to insert incorporate lock: {}", e)) - })?; - } + + tx.execute( + "INSERT OR REPLACE INTO incorporate_locks (stem, release) VALUES (?1, ?2)", + rusqlite::params![stem, release], + ) + .map_err(|e| ImageError::Database(format!("Failed to insert incorporate lock: {}", e)))?; + tx.commit().map_err(|e| { ImageError::Database(format!("Failed to commit incorporate lock: {}", e)) })?; Ok(()) } - /// Get a manifest from the catalog + /// Get a manifest from the catalog. + /// First checks the local manifest cache on disk, then falls back to repository fetch. + /// Note: active.db does NOT store manifest blobs - manifests are served from the repository. pub fn get_manifest_from_catalog( &self, fmri: &crate::fmri::Fmri, ) -> Result> { - let catalog = ImageCatalog::new( + // Helper to URL-encode filename components + fn url_encode(s: &str) -> String { + s.chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(), + ' ' => "+".to_string(), + _ => { + let mut buf = [0u8; 4]; + let bytes = c.encode_utf8(&mut buf).as_bytes(); + bytes.iter().map(|b| format!("%{:02X}", b)).collect() + } + }) + .collect() + } + + // Check local manifest cache on disk + let publisher = fmri.publisher.as_deref().unwrap_or(""); + let manifest_dir = self.manifest_dir().join(publisher); + + let stem_encoded = url_encode(fmri.stem()); + let version_encoded = url_encode(&fmri.version()); + let manifest_path = manifest_dir.join(format!("{}@{}.p5m", stem_encoded, version_encoded)); + + if manifest_path.exists() { + let content = fs::read_to_string(&manifest_path)?; + let manifest = crate::actions::Manifest::parse_string(content) + .map_err(|e| ImageError::Database(format!("Failed to parse manifest: {}", e)))?; + return Ok(Some(manifest)); + } + + // Check catalog shards for a minimal manifest + let catalog = crate::image::catalog::ImageCatalog::new( self.catalog_dir(), - self.catalog_db_path(), - self.obsoleted_db_path(), + self.active_db_path(), + self.obsolete_db_path(), ); - catalog.get_manifest(fmri).map_err(|e| { - ImageError::Database(format!("Failed to get manifest from catalog: {}", e)) - }) + if let Ok(Some(manifest)) = catalog.get_manifest(fmri) { + return Ok(Some(manifest)); + } + + // Fall back to repository fetch + match self.get_manifest_from_repository(fmri) { + Ok(manifest) => Ok(Some(manifest)), + Err(_) => Ok(None), + } } /// Fetch a full manifest for the given FMRI directly from its repository origin. diff --git a/libips/src/repository/file_backend.rs b/libips/src/repository/file_backend.rs index 9d451ff..3a7e674 100644 --- a/libips/src/repository/file_backend.rs +++ b/libips/src/repository/file_backend.rs @@ -1899,11 +1899,25 @@ impl WritableRepository for FileBackend { if !no_catalog { info!("Rebuilding catalog..."); self.rebuild_catalog(&pub_name, true)?; + + // Build SQLite catalog shards (active.db, obsolete.db, fts.db) + info!("Building catalog shards..."); + let catalog_dir = Self::construct_catalog_path(&self.path, &pub_name); + let shard_dir = self.shard_dir(&pub_name); + crate::repository::sqlite_catalog::build_shards( + &catalog_dir, + &pub_name, + &shard_dir, + ) + .map_err(|e| { + RepositoryError::Other(format!("Failed to build catalog shards: {}", e.message)) + })?; } if !no_index { - info!("Rebuilding search index..."); - self.build_search_index(&pub_name)?; + // FTS index is now built as part of catalog shards (fts.db) + // No separate index building needed + info!("Search index built as part of catalog shards (fts.db)"); } } @@ -1929,11 +1943,25 @@ impl WritableRepository for FileBackend { if !no_catalog { info!("Refreshing catalog..."); self.rebuild_catalog(&pub_name, true)?; + + // Build SQLite catalog shards (active.db, obsolete.db, fts.db) + info!("Building catalog shards..."); + let catalog_dir = Self::construct_catalog_path(&self.path, &pub_name); + let shard_dir = self.shard_dir(&pub_name); + crate::repository::sqlite_catalog::build_shards( + &catalog_dir, + &pub_name, + &shard_dir, + ) + .map_err(|e| { + RepositoryError::Other(format!("Failed to build catalog shards: {}", e.message)) + })?; } if !no_index { - info!("Refreshing search index..."); - self.build_search_index(&pub_name)?; + // FTS index is now built as part of catalog shards (fts.db) + // No separate index building needed + info!("Search index built as part of catalog shards (fts.db)"); } } @@ -2166,6 +2194,13 @@ impl FileBackend { base_path.join("publisher").join(publisher).join("catalog") } + /// Helper method to construct a shard directory path for catalog v2 shards + /// + /// Format: base_path/publisher/publisher_name/catalog2 + pub fn shard_dir(&self, publisher: &str) -> PathBuf { + self.path.join("publisher").join(publisher).join("catalog2") + } + /// Helper method to construct a manifest path consistently /// /// Format: base_path/publisher/publisher_name/pkg/stem/encoded_version diff --git a/libips/src/repository/mod.rs b/libips/src/repository/mod.rs index a14242d..7ce9657 100644 --- a/libips/src/repository/mod.rs +++ b/libips/src/repository/mod.rs @@ -168,60 +168,21 @@ impl From for RepositoryError { } } -// Implement From for redb error types -impl From for RepositoryError { - fn from(err: redb::Error) -> Self { +// Implement From for rusqlite error types +impl From for RepositoryError { + fn from(err: rusqlite::Error) -> Self { RepositoryError::Other(format!("Database error: {}", err)) } } -impl From for RepositoryError { - fn from(err: redb::DatabaseError) -> Self { - RepositoryError::Other(format!("Database error: {}", err)) - } -} - -impl From for RepositoryError { - fn from(err: redb::TransactionError) -> Self { - RepositoryError::Other(format!("Transaction error: {}", err)) - } -} - -impl From for RepositoryError { - fn from(err: redb::TableError) -> Self { - RepositoryError::Other(format!("Table error: {}", err)) - } -} - -impl From for RepositoryError { - fn from(err: redb::StorageError) -> Self { - RepositoryError::Other(format!("Storage error: {}", err)) - } -} - -impl From for RepositoryError { - fn from(err: redb::CommitError) -> Self { - RepositoryError::Other(format!("Commit error: {}", err)) - } -} - -impl From for RepositoryError { - fn from(err: bincode::error::DecodeError) -> Self { - RepositoryError::Other(format!("Serialization error: {}", err)) - } -} - -impl From for RepositoryError { - fn from(err: bincode::error::EncodeError) -> Self { - RepositoryError::Other(format!("Serialization error: {}", err)) - } -} pub mod catalog; mod catalog_writer; pub(crate) mod file_backend; mod obsoleted; pub mod progress; mod rest_backend; +pub mod shard_sync; +pub mod sqlite_catalog; #[cfg(test)] mod tests; diff --git a/libips/src/repository/obsoleted.rs b/libips/src/repository/obsoleted.rs index 2227a0a..1b62097 100644 --- a/libips/src/repository/obsoleted.rs +++ b/libips/src/repository/obsoleted.rs @@ -1,11 +1,11 @@ use crate::fmri::Fmri; +use crate::repository::sqlite_catalog::OBSOLETED_INDEX_SCHEMA; use crate::repository::{RepositoryError, Result}; use chrono::{DateTime, Duration as ChronoDuration, Utc}; use miette::Diagnostic; -use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition}; use regex::Regex; +use rusqlite::{Connection, OpenFlags}; use serde::{Deserialize, Serialize}; -use serde_cbor; use serde_json; use sha2::Digest; use std::fs; @@ -176,54 +176,12 @@ impl From for ObsoletedPackageError { } } -impl From for ObsoletedPackageError { - fn from(err: redb::Error) -> Self { +impl From for ObsoletedPackageError { + fn from(err: rusqlite::Error) -> Self { ObsoletedPackageError::DatabaseError(err.to_string()) } } -impl From for ObsoletedPackageError { - fn from(err: redb::DatabaseError) -> Self { - ObsoletedPackageError::DatabaseError(err.to_string()) - } -} - -impl From for ObsoletedPackageError { - fn from(err: redb::TransactionError) -> Self { - ObsoletedPackageError::DatabaseError(err.to_string()) - } -} - -impl From for ObsoletedPackageError { - fn from(err: redb::TableError) -> Self { - ObsoletedPackageError::DatabaseError(err.to_string()) - } -} - -impl From for ObsoletedPackageError { - fn from(err: redb::StorageError) -> Self { - ObsoletedPackageError::DatabaseError(err.to_string()) - } -} - -impl From for ObsoletedPackageError { - fn from(err: redb::CommitError) -> Self { - ObsoletedPackageError::DatabaseError(err.to_string()) - } -} - -impl From for ObsoletedPackageError { - fn from(err: bincode::error::EncodeError) -> Self { - ObsoletedPackageError::SerializationError(err.to_string()) - } -} - -impl From for ObsoletedPackageError { - fn from(err: bincode::error::DecodeError) -> Self { - ObsoletedPackageError::SerializationError(err.to_string()) - } -} - // Implement From for RepositoryError to allow conversion // This makes it easier to use ObsoletedPackageError with the existing Result type impl From for RepositoryError { @@ -325,19 +283,11 @@ impl ObsoletedPackageKey { } } -// Table definitions for the redb database -// Table for mapping FMRI directly to metadata -static FMRI_TO_METADATA_TABLE: TableDefinition<&[u8], &[u8]> = - TableDefinition::new("fmri_to_metadata"); -// Table for mapping content hash to manifest (for non-NULL_HASH entries) -static HASH_TO_MANIFEST_TABLE: TableDefinition<&str, &str> = - TableDefinition::new("hash_to_manifest"); - -/// Index of obsoleted packages using redb for faster lookups and content-addressable storage +/// Index of obsoleted packages using SQLite for faster lookups and content-addressable storage #[derive(Debug)] -struct RedbObsoletedPackageIndex { - /// The redb database - db: Database, +struct SqliteObsoletedPackageIndex { + /// Path to the SQLite database file + db_path: PathBuf, /// Last time the index was accessed last_accessed: Instant, /// Whether the index is dirty and needs to be rebuilt @@ -346,26 +296,18 @@ struct RedbObsoletedPackageIndex { max_age: Duration, } -impl RedbObsoletedPackageIndex { - /// Create a new RedbObsoletedPackageIndex +impl SqliteObsoletedPackageIndex { + /// Create a new SqliteObsoletedPackageIndex fn new>(base_path: P) -> Result { - let db_path = base_path.as_ref().join("index.redb"); - debug!("Creating redb database at {}", db_path.display()); + let db_path = base_path.as_ref().join("index.db"); + debug!("Creating SQLite database at {}", db_path.display()); - // Create the database - let db = Database::create(&db_path)?; - - // Create the tables if they don't exist - let write_txn = db.begin_write()?; - { - // Create the new table for direct FMRI to metadata mapping - write_txn.open_table(FMRI_TO_METADATA_TABLE)?; - write_txn.open_table(HASH_TO_MANIFEST_TABLE)?; - } - write_txn.commit()?; + // Create the database and tables + let conn = Connection::open(&db_path)?; + conn.execute_batch(OBSOLETED_INDEX_SCHEMA)?; Ok(Self { - db, + db_path, last_accessed: Instant::now(), dirty: false, max_age: Duration::from_secs(300), // 5 minutes @@ -377,12 +319,12 @@ impl RedbObsoletedPackageIndex { self.dirty || self.last_accessed.elapsed() > self.max_age } - /// Create an empty temporary file-based RedbObsoletedPackageIndex + /// Create an empty temporary file-based SqliteObsoletedPackageIndex /// /// This is used as a fallback when the database creation fails. /// It creates a database in a temporary directory that can be used temporarily. fn empty() -> Self { - debug!("Creating empty temporary file-based redb database"); + debug!("Creating empty temporary file-based SQLite database"); // Create a temporary directory let temp_dir = tempfile::tempdir().unwrap_or_else(|e| { @@ -391,50 +333,44 @@ impl RedbObsoletedPackageIndex { }); // Create a database file in the temporary directory - let db_path = temp_dir.path().join("empty.redb"); + let db_path = temp_dir.path().join("empty.db"); - // Create the database - let db = Database::create(&db_path).unwrap_or_else(|e| { + // Create the database and tables + let conn = Connection::open(&db_path).unwrap_or_else(|e| { error!("Failed to create temporary database: {}", e); panic!("Failed to create temporary database: {}", e); }); - // Create the tables - let write_txn = db.begin_write().unwrap(); - { - // Create the new table for direct FMRI to metadata mapping - let _ = write_txn.open_table(FMRI_TO_METADATA_TABLE).unwrap(); - let _ = write_txn.open_table(HASH_TO_MANIFEST_TABLE).unwrap(); - } - write_txn.commit().unwrap(); + conn.execute_batch(OBSOLETED_INDEX_SCHEMA).unwrap(); Self { - db, + db_path, last_accessed: Instant::now(), dirty: false, max_age: Duration::from_secs(300), // 5 minutes } } - /// Open an existing RedbObsoletedPackageIndex + /// Open an existing SqliteObsoletedPackageIndex fn open>(base_path: P) -> Result { - let db_path = base_path.as_ref().join("index.redb"); - debug!("Opening redb database at {}", db_path.display()); + let db_path = base_path.as_ref().join("index.db"); + debug!("Opening SQLite database at {}", db_path.display()); - // Open the database - let db = Database::open(&db_path)?; + // Open the database (creating tables if they don't exist) + let conn = Connection::open(&db_path)?; + conn.execute_batch(OBSOLETED_INDEX_SCHEMA)?; Ok(Self { - db, + db_path, last_accessed: Instant::now(), dirty: false, max_age: Duration::from_secs(300), // 5 minutes }) } - /// Create or open a RedbObsoletedPackageIndex + /// Create or open a SqliteObsoletedPackageIndex fn create_or_open>(base_path: P) -> Result { - let db_path = base_path.as_ref().join("index.redb"); + let db_path = base_path.as_ref().join("index.db"); if db_path.exists() { Self::open(base_path) @@ -464,69 +400,46 @@ impl RedbObsoletedPackageIndex { metadata.content_hash.clone() }; - // Use the FMRI string directly as the key - let key_bytes = metadata.fmri.as_bytes(); + // Serialize obsoleted_by as JSON string (or NULL if None) + let obsoleted_by_json = metadata + .obsoleted_by + .as_ref() + .map(|obs| serde_json::to_string(obs)) + .transpose()?; - let metadata_bytes = match serde_cbor::to_vec(metadata) { - Ok(bytes) => bytes, - Err(e) => { - error!("Failed to serialize metadata with CBOR: {}", e); - return Err(ObsoletedPackageError::SerializationError(format!( - "Failed to serialize metadata with CBOR: {}", - e - )) - .into()); - } - }; + let mut conn = Connection::open(&self.db_path)?; + let tx = conn.transaction()?; - // Begin write transaction - let write_txn = match self.db.begin_write() { - Ok(txn) => txn, - Err(e) => { - error!("Failed to begin write transaction: {}", e); - return Err(e.into()); - } - }; + // Insert into obsoleted_packages table + tx.execute( + "INSERT OR REPLACE INTO obsoleted_packages ( + fmri, publisher, stem, version, status, obsolescence_date, + deprecation_message, obsoleted_by, metadata_version, content_hash + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + rusqlite::params![ + &metadata.fmri, + &key.publisher, + &key.stem, + &key.version, + &metadata.status, + &metadata.obsolescence_date, + metadata.deprecation_message.as_deref(), + obsoleted_by_json.as_deref(), + metadata.metadata_version, + &content_hash, + ], + )?; - { - // Open the tables - let mut fmri_to_metadata = match write_txn.open_table(FMRI_TO_METADATA_TABLE) { - Ok(table) => table, - Err(e) => { - error!("Failed to open FMRI_TO_METADATA_TABLE: {}", e); - return Err(e.into()); - } - }; - - let mut hash_to_manifest = match write_txn.open_table(HASH_TO_MANIFEST_TABLE) { - Ok(table) => table, - Err(e) => { - error!("Failed to open HASH_TO_MANIFEST_TABLE: {}", e); - return Err(e.into()); - } - }; - - // Insert the metadata directly with FMRI as the key - // This is the new approach that eliminates the intermediate hash lookup - if let Err(e) = fmri_to_metadata.insert(key_bytes, metadata_bytes.as_slice()) { - error!("Failed to insert into FMRI_TO_METADATA_TABLE: {}", e); - return Err(e.into()); - } - - // Only store the manifest if it's not a NULL_HASH entry - // For NULL_HASH entries, a minimal manifest will be generated when requested - if content_hash != NULL_HASH { - if let Err(e) = hash_to_manifest.insert(content_hash.as_str(), manifest) { - error!("Failed to insert into HASH_TO_MANIFEST_TABLE: {}", e); - return Err(e.into()); - } - } + // Only store the manifest if it's not a NULL_HASH entry + // For NULL_HASH entries, a minimal manifest will be generated when requested + if content_hash != NULL_HASH { + tx.execute( + "INSERT OR REPLACE INTO obsoleted_manifests (content_hash, manifest) VALUES (?1, ?2)", + rusqlite::params![&content_hash, manifest], + )?; } - if let Err(e) = write_txn.commit() { - error!("Failed to commit transaction: {}", e); - return Err(e.into()); - } + tx.commit()?; debug!("Successfully added entry to index: {}", metadata.fmri); Ok(()) @@ -536,32 +449,25 @@ impl RedbObsoletedPackageIndex { fn remove_entry(&self, key: &ObsoletedPackageKey) -> Result { // Use the FMRI string directly as the key let fmri = key.to_fmri_string(); - let key_bytes = fmri.as_bytes(); - // First, check if the key exists in the new table - let exists_in_new_table = { - let read_txn = self.db.begin_read()?; - let fmri_to_metadata = read_txn.open_table(FMRI_TO_METADATA_TABLE)?; + let conn = Connection::open(&self.db_path)?; - fmri_to_metadata.get(key_bytes)?.is_some() - }; + // Check if the entry exists + let exists: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM obsoleted_packages WHERE fmri = ?1)", + rusqlite::params![&fmri], + |row| row.get(0), + )?; - // If the key doesn't exist in either table, return early - if !exists_in_new_table { + if !exists { return Ok(false); } - // Now perform the actual removal - let write_txn = self.db.begin_write()?; - { - // Remove the entry from the new table - if exists_in_new_table { - let mut fmri_to_metadata = write_txn.open_table(FMRI_TO_METADATA_TABLE)?; - fmri_to_metadata.remove(key_bytes)?; - } - } - - write_txn.commit()?; + // Remove the entry + conn.execute( + "DELETE FROM obsoleted_packages WHERE fmri = ?1", + rusqlite::params![&fmri], + )?; Ok(true) } @@ -573,43 +479,61 @@ impl RedbObsoletedPackageIndex { ) -> Result> { // Use the FMRI string directly as the key let fmri = key.to_fmri_string(); - let key_bytes = fmri.as_bytes(); - // First, try to get the metadata directly from the new table - let metadata_result = { - let read_txn = self.db.begin_read()?; - let fmri_to_metadata = read_txn.open_table(FMRI_TO_METADATA_TABLE)?; + let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; - // Get the metadata bytes - match fmri_to_metadata.get(key_bytes)? { - Some(bytes) => { - // Convert to owned bytes before the transaction is dropped - let metadata_bytes = bytes.value().to_vec(); - // Try to deserialize the metadata - match serde_cbor::from_slice::(&metadata_bytes) { - Ok(metadata) => Some(metadata), - Err(e) => { - warn!( - "Failed to deserialize metadata from FMRI_TO_METADATA_TABLE with CBOR: {}", - e - ); - None - } - } - } - None => None, - } + // Try to get the metadata from the database + let metadata_result = match conn.query_row( + "SELECT fmri, status, obsolescence_date, deprecation_message, + obsoleted_by, metadata_version, content_hash + FROM obsoleted_packages WHERE fmri = ?1", + rusqlite::params![&fmri], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, u32>(5)?, + row.get::<_, String>(6)?, + )) + }, + ) { + Ok(result) => Some(result), + Err(rusqlite::Error::QueryReturnedNoRows) => None, + Err(e) => return Err(e.into()), }; - // If we found the metadata in the new table, use it - if let Some(metadata) = metadata_result { - // Get the content hash from the metadata - let content_hash = metadata.content_hash.clone(); + if let Some(( + fmri, + status, + obsolescence_date, + deprecation_message, + obsoleted_by_json, + metadata_version, + content_hash, + )) = metadata_result + { + // Deserialize obsoleted_by from JSON if present + let obsoleted_by = obsoleted_by_json + .as_ref() + .map(|json| serde_json::from_str::>(json)) + .transpose()?; + + let metadata = ObsoletedPackageMetadata { + fmri, + status, + obsolescence_date, + deprecation_message, + obsoleted_by, + metadata_version, + content_hash: content_hash.clone(), + }; // For NULL_HASH entries, generate a minimal manifest let manifest_str = if content_hash == NULL_HASH { // Generate a minimal manifest for NULL_HASH entries - // Construct an FMRI string from the metadata format!( r#"{{ "attributes": [ @@ -631,13 +555,13 @@ impl RedbObsoletedPackageIndex { ) } else { // For non-NULL_HASH entries, get the manifest from the database - let read_txn = self.db.begin_read()?; - let hash_to_manifest = read_txn.open_table(HASH_TO_MANIFEST_TABLE)?; - - // Get the manifest string - match hash_to_manifest.get(content_hash.as_str())? { - Some(manifest) => manifest.value().to_string(), - None => { + match conn.query_row( + "SELECT manifest FROM obsoleted_manifests WHERE content_hash = ?1", + rusqlite::params![&content_hash], + |row| row.get::<_, String>(0), + ) { + Ok(manifest) => manifest, + Err(rusqlite::Error::QueryReturnedNoRows) => { warn!( "Manifest not found for content hash: {}, generating minimal manifest", content_hash @@ -663,6 +587,7 @@ impl RedbObsoletedPackageIndex { metadata.fmri ) } + Err(e) => return Err(e.into()), } }; Ok(Some((metadata, manifest_str))) @@ -676,119 +601,146 @@ impl RedbObsoletedPackageIndex { &self, ) -> Result> { let mut entries = Vec::new(); - let mut processed_keys = std::collections::HashSet::new(); - // First, collect all entries from the new table - { - let read_txn = self.db.begin_read()?; - let fmri_to_metadata = read_txn.open_table(FMRI_TO_METADATA_TABLE)?; + let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; - let mut iter = fmri_to_metadata.iter()?; + let mut stmt = conn.prepare( + "SELECT fmri, publisher, stem, version, status, obsolescence_date, + deprecation_message, obsoleted_by, metadata_version, content_hash + FROM obsoleted_packages", + )?; - while let Some(entry) = iter.next() { - let (key_bytes, metadata_bytes) = entry?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, String>(5)?, + row.get::<_, Option>(6)?, + row.get::<_, Option>(7)?, + row.get::<_, u32>(8)?, + row.get::<_, String>(9)?, + )) + })?; - // Convert to owned types before the transaction is dropped - let key_data = key_bytes.value().to_vec(); - let metadata_data = metadata_bytes.value().to_vec(); + for row_result in rows { + let ( + fmri, + _publisher, + _stem, + _version, + status, + obsolescence_date, + deprecation_message, + obsoleted_by_json, + metadata_version, + content_hash, + ) = match row_result { + Ok(r) => r, + Err(e) => { + warn!("Failed to read row: {}", e); + continue; + } + }; - // Convert key bytes to string and parse as FMRI - let fmri_str = match std::str::from_utf8(&key_data) { - Ok(s) => s, - Err(e) => { - warn!("Failed to convert key bytes to string: {}", e); - continue; - } - }; + // Parse the FMRI string to create an ObsoletedPackageKey + let key = match ObsoletedPackageKey::from_fmri_string(&fmri) { + Ok(key) => key, + Err(e) => { + warn!("Failed to parse FMRI string: {}", e); + continue; + } + }; - // Parse the FMRI string to create an ObsoletedPackageKey - let key = match ObsoletedPackageKey::from_fmri_string(fmri_str) { - Ok(key) => key, - Err(e) => { - warn!("Failed to parse FMRI string: {}", e); - continue; - } - }; + // Deserialize obsoleted_by from JSON if present + let obsoleted_by = match obsoleted_by_json + .as_ref() + .map(|json| serde_json::from_str::>(json)) + .transpose() + { + Ok(obs) => obs, + Err(e) => { + warn!("Failed to deserialize obsoleted_by JSON: {}", e); + continue; + } + }; - let metadata: ObsoletedPackageMetadata = match serde_cbor::from_slice( - &metadata_data, + let metadata = ObsoletedPackageMetadata { + fmri: fmri.clone(), + status, + obsolescence_date, + deprecation_message, + obsoleted_by, + metadata_version, + content_hash: content_hash.clone(), + }; + + // For NULL_HASH entries, generate a minimal manifest + let manifest_str = if content_hash == NULL_HASH { + // Generate a minimal manifest for NULL_HASH entries + format!( + r#"{{ + "attributes": [ + {{ + "key": "pkg.fmri", + "values": [ + "{}" + ] + }}, + {{ + "key": "pkg.obsolete", + "values": [ + "true" + ] + }} + ] +}}"#, + fmri + ) + } else { + // For non-NULL_HASH entries, get the manifest from the database + match conn.query_row( + "SELECT manifest FROM obsoleted_manifests WHERE content_hash = ?1", + rusqlite::params![&content_hash], + |row| row.get::<_, String>(0), ) { - Ok(metadata) => metadata, - Err(e) => { + Ok(manifest) => manifest, + Err(rusqlite::Error::QueryReturnedNoRows) => { warn!( - "Failed to deserialize metadata from FMRI_TO_METADATA_TABLE with CBOR: {}", - e + "Manifest not found for content hash: {}, generating minimal manifest", + content_hash ); + // Generate a minimal manifest as a fallback + format!( + r#"{{ + "attributes": [ + {{ + "key": "pkg.fmri", + "values": [ + "{}" + ] + }}, + {{ + "key": "pkg.obsolete", + "values": [ + "true" + ] + }} + ] +}}"#, + fmri + ) + } + Err(e) => { + warn!("Failed to get manifest for content hash: {}", e); continue; } - }; + } + }; - // Add the key to the set of processed keys - processed_keys.insert(key_data); - - // Get the content hash from the metadata - let content_hash = metadata.content_hash.clone(); - - // For NULL_HASH entries, generate a minimal manifest - let manifest_str = if content_hash == NULL_HASH { - // Generate a minimal manifest for NULL_HASH entries - format!( - r#"{{ - "attributes": [ - {{ - "key": "pkg.fmri", - "values": [ - "{}" - ] - }}, - {{ - "key": "pkg.obsolete", - "values": [ - "true" - ] - }} - ] -}}"#, - metadata.fmri - ) - } else { - // For non-NULL_HASH entries, get the manifest from the database - let hash_to_manifest = read_txn.open_table(HASH_TO_MANIFEST_TABLE)?; - - // Get the manifest string - match hash_to_manifest.get(content_hash.as_str())? { - Some(manifest) => manifest.value().to_string(), - None => { - warn!( - "Manifest not found for content hash: {}, generating minimal manifest", - content_hash - ); - // Generate a minimal manifest as a fallback - format!( - r#"{{ - "attributes": [ - {{ - "key": "pkg.fmri", - "values": [ - "{}" - ] - }}, - {{ - "key": "pkg.obsolete", - "values": [ - "true" - ] - }} - ] -}}"#, - metadata.fmri - ) - } - } - }; - - entries.push((key, metadata, manifest_str)); - } + entries.push((key, metadata, manifest_str)); } Ok(entries) @@ -858,86 +810,26 @@ impl RedbObsoletedPackageIndex { /// Clear the index fn clear(&self) -> Result<()> { - // Begin a writing transaction - let write_txn = self.db.begin_write()?; - { - // Clear all tables by removing all entries - // Since redb doesn't have a clear() method, we need to iterate and remove each key + let conn = Connection::open(&self.db_path)?; - // Clear hash_to_manifest table - { - let mut hash_to_manifest = write_txn.open_table(FMRI_TO_METADATA_TABLE)?; - let keys_to_remove = { - // First, collect all keys in a separate scope - let read_txn = self.db.begin_read()?; - let hash_to_manifest_read = read_txn.open_table(FMRI_TO_METADATA_TABLE)?; - let mut keys = Vec::new(); - let mut iter = hash_to_manifest_read.iter()?; - while let Some(entry) = iter.next() { - let (key, _) = entry?; - keys.push(key.value().to_vec()); - } - keys - }; - - // Then remove all keys - for key in keys_to_remove { - hash_to_manifest.remove(key.as_slice())?; - } - } - - // Clear hash_to_manifest table - { - let mut hash_to_manifest = write_txn.open_table(HASH_TO_MANIFEST_TABLE)?; - let keys_to_remove = { - // First, collect all keys in a separate scope - let read_txn = self.db.begin_read()?; - let hash_to_manifest_read = read_txn.open_table(HASH_TO_MANIFEST_TABLE)?; - let mut keys = Vec::new(); - let mut iter = hash_to_manifest_read.iter()?; - while let Some(entry) = iter.next() { - let (key, _) = entry?; - keys.push(key.value().to_string()); - } - keys - }; - - // Then remove all keys - for key in keys_to_remove { - hash_to_manifest.remove(key.as_str())?; - } - } - } - write_txn.commit()?; + // Clear both tables + conn.execute("DELETE FROM obsoleted_packages", [])?; + conn.execute("DELETE FROM obsoleted_manifests", [])?; Ok(()) } /// Get the number of entries in the index fn len(&self) -> Result { - // Begin a read transaction - let read_txn = self.db.begin_read()?; + let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?; - // Open the fmri_to_hash table - let fmri_to_hash = read_txn.open_table(FMRI_TO_METADATA_TABLE)?; + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM obsoleted_packages", + [], + |row| row.get(0), + )?; - // Count the entries - let mut count = 0; - let mut iter = fmri_to_hash.iter()?; - - // Iterate through all entries and count them - while let Some(entry_result) = iter.next() { - // Just check if the entry exists, we don't need to access its value - entry_result?; - count += 1; - } - - // Drop the iterator and table before returning - drop(iter); - drop(fmri_to_hash); - drop(read_txn); - - Ok(count) + Ok(count as usize) } /// Check if the index is empty @@ -1033,8 +925,8 @@ impl ObsoletedPackageMetadata { pub struct ObsoletedPackageManager { /// Base path for obsoleted packages base_path: PathBuf, - /// Index of obsoleted packages for faster lookups using redb - index: RwLock, + /// Index of obsoleted packages for faster lookups using SQLite + index: RwLock, } impl ObsoletedPackageManager { @@ -1075,14 +967,14 @@ impl ObsoletedPackageManager { let base_path = repo_path.as_ref().join("obsoleted"); let index = { - // Create or open the redb-based index - let redb_index = - RedbObsoletedPackageIndex::create_or_open(&base_path).unwrap_or_else(|e| { - // Log the error and create an empty redb index - error!("Failed to create or open redb-based index: {}", e); - RedbObsoletedPackageIndex::empty() + // Create or open the SQLite-based index + let sqlite_index = + SqliteObsoletedPackageIndex::create_or_open(&base_path).unwrap_or_else(|e| { + // Log the error and create an empty SQLite index + error!("Failed to create or open SQLite-based index: {}", e); + SqliteObsoletedPackageIndex::empty() }); - RwLock::new(redb_index) + RwLock::new(sqlite_index) }; Self { base_path, index } diff --git a/libips/src/repository/shard_sync.rs b/libips/src/repository/shard_sync.rs new file mode 100644 index 0000000..8a598d2 --- /dev/null +++ b/libips/src/repository/shard_sync.rs @@ -0,0 +1,164 @@ +// This Source Code Form is subject to the terms of +// the Mozilla Public License, v. 2.0. If a copy of the +// MPL was not distributed with this file, You can +// obtain one at https://mozilla.org/MPL/2.0/. + +//! Client-side shard synchronization. +//! +//! Downloads catalog shards from the repository server and verifies their integrity. + +use crate::repository::sqlite_catalog::{ShardIndex, ShardEntry}; +use miette::Diagnostic; +use sha2::{Digest, Sha256}; +use std::fs; +use std::path::{Path, PathBuf}; +use thiserror::Error; + +#[derive(Debug, Error, Diagnostic)] +#[error("Shard sync error: {message}")] +#[diagnostic(code(ips::shard_sync_error))] +pub struct ShardSyncError { + pub message: String, +} + +impl ShardSyncError { + fn new(msg: impl Into) -> Self { + Self { + message: msg.into(), + } + } +} + +impl From for ShardSyncError { + fn from(e: reqwest::Error) -> Self { + Self::new(format!("HTTP error: {}", e)) + } +} + +impl From for ShardSyncError { + fn from(e: std::io::Error) -> Self { + Self::new(format!("IO error: {}", e)) + } +} + +impl From for ShardSyncError { + fn from(e: serde_json::Error) -> Self { + Self::new(format!("JSON error: {}", e)) + } +} + +/// Synchronize catalog shards from a repository origin. +/// +/// Downloads the shard index from `{origin_url}/{publisher}/catalog/2/catalog.attrs`, +/// compares hashes with local copies, and downloads only changed shards. +/// +/// # Arguments +/// * `publisher` - Publisher name +/// * `origin_url` - Repository origin URL (e.g., "https://pkg.example.com") +/// * `local_shard_dir` - Local directory to store shards +/// * `download_obsolete` - Whether to download obsolete.db (default: false) +pub fn sync_shards( + publisher: &str, + origin_url: &str, + local_shard_dir: &Path, + download_obsolete: bool, +) -> Result<(), ShardSyncError> { + // Ensure local directory exists + fs::create_dir_all(local_shard_dir)?; + + // Fetch shard index + let index_url = format!("{}/{}/catalog/2/catalog.attrs", origin_url, publisher); + let client = reqwest::blocking::Client::new(); + let response = client.get(&index_url).send()?; + + if !response.status().is_success() { + return Err(ShardSyncError::new(format!( + "Failed to fetch shard index: HTTP {}", + response.status() + ))); + } + + let index: ShardIndex = response.json()?; + + // List of shards to sync + let shards_to_sync = if download_obsolete { + vec!["active.db", "fts.db", "obsolete.db"] + } else { + vec!["active.db", "fts.db"] + }; + + // Download each shard if needed + for shard_name in shards_to_sync { + let Some(shard_entry) = index.shards.get(shard_name) else { + tracing::warn!("Shard {} not found in index", shard_name); + continue; + }; + + let local_path = local_shard_dir.join(shard_name); + + // Check if local copy exists and matches hash + let needs_download = if local_path.exists() { + match compute_sha256(&local_path) { + Ok(local_hash) => local_hash != shard_entry.sha256, + Err(_) => true, // Error reading local file, re-download + } + } else { + true + }; + + if !needs_download { + tracing::debug!("Shard {} is up to date", shard_name); + continue; + } + + // Download shard + tracing::info!("Downloading shard {} from {}", shard_name, origin_url); + let shard_url = format!( + "{}/{}/catalog/2/{}", + origin_url, publisher, &shard_entry.sha256 + ); + let mut response = client.get(&shard_url).send()?; + + if !response.status().is_success() { + return Err(ShardSyncError::new(format!( + "Failed to download shard {}: HTTP {}", + shard_name, + response.status() + ))); + } + + // Write to temporary file + let temp_path = local_shard_dir.join(format!("{}.tmp", shard_name)); + let mut file = fs::File::create(&temp_path)?; + response.copy_to(&mut file)?; + drop(file); + + // Verify SHA-256 + let downloaded_hash = compute_sha256(&temp_path)?; + if downloaded_hash != shard_entry.sha256 { + fs::remove_file(&temp_path)?; + return Err(ShardSyncError::new(format!( + "SHA-256 mismatch for {}: expected {}, got {}", + shard_name, shard_entry.sha256, downloaded_hash + ))); + } + + // Atomic rename + fs::rename(&temp_path, &local_path)?; + tracing::info!("Successfully downloaded {}", shard_name); + } + + // Write local copy of index for future comparisons + let index_json = serde_json::to_string_pretty(&index)?; + fs::write(local_shard_dir.join("catalog.attrs"), index_json)?; + + Ok(()) +} + +/// Compute SHA-256 hash of a file. +fn compute_sha256(path: &Path) -> Result { + let bytes = fs::read(path)?; + let mut hasher = Sha256::new(); + hasher.update(&bytes); + Ok(format!("{:x}", hasher.finalize())) +} diff --git a/libips/src/repository/sqlite_catalog.rs b/libips/src/repository/sqlite_catalog.rs new file mode 100644 index 0000000..ff0fcb7 --- /dev/null +++ b/libips/src/repository/sqlite_catalog.rs @@ -0,0 +1,484 @@ +// This Source Code Form is subject to the terms of +// the Mozilla Public License, v. 2.0. If a copy of the +// MPL was not distributed with this file, You can +// obtain one at https://mozilla.org/MPL/2.0/. + +//! SQLite catalog shard generation and population. +//! +//! This module defines all SQLite schemas used by the IPS system and provides +//! functions to build pre-built catalog shards for distribution via the +//! catalog/2 endpoint. + +use crate::actions::Manifest; +use crate::fmri::Fmri; +use crate::repository::catalog::CatalogManager; +use base64::Engine as _; +use miette::Diagnostic; +use rusqlite::Connection; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::BTreeMap; +use std::fs; +use std::path::Path; +use thiserror::Error; + +/// Schema for active.db - contains non-obsolete packages and their dependencies. +/// No manifest blobs stored; manifests are fetched from repository on demand. +pub const ACTIVE_SCHEMA: &str = r#" +CREATE TABLE IF NOT EXISTS packages ( + stem TEXT NOT NULL, + version TEXT NOT NULL, + publisher TEXT NOT NULL, + fmri TEXT GENERATED ALWAYS AS ( + 'pkg://' || publisher || '/' || stem || '@' || version + ) STORED, + PRIMARY KEY (stem, version, publisher) +); +CREATE INDEX IF NOT EXISTS idx_packages_fmri ON packages(fmri); +CREATE INDEX IF NOT EXISTS idx_packages_stem ON packages(stem); + +CREATE TABLE IF NOT EXISTS dependencies ( + pkg_stem TEXT NOT NULL, + pkg_version TEXT NOT NULL, + pkg_publisher TEXT NOT NULL, + dep_type TEXT NOT NULL, + dep_stem TEXT NOT NULL, + dep_version TEXT, + PRIMARY KEY (pkg_stem, pkg_version, pkg_publisher, dep_type, dep_stem) +); +CREATE INDEX IF NOT EXISTS idx_deps_pkg ON dependencies(pkg_stem, pkg_version, pkg_publisher); + +CREATE TABLE IF NOT EXISTS incorporate_locks ( + stem TEXT NOT NULL PRIMARY KEY, + release TEXT NOT NULL +); +"#; + +/// Schema for obsolete.db - client-side shard for obsoleted packages. +pub const OBSOLETE_SCHEMA: &str = r#" +CREATE TABLE IF NOT EXISTS obsolete_packages ( + publisher TEXT NOT NULL, + stem TEXT NOT NULL, + version TEXT NOT NULL, + fmri TEXT GENERATED ALWAYS AS ( + 'pkg://' || publisher || '/' || stem || '@' || version + ) STORED, + PRIMARY KEY (publisher, stem, version) +); +CREATE INDEX IF NOT EXISTS idx_obsolete_fmri ON obsolete_packages(fmri); +"#; + +/// Schema for fts.db - full-text search index. +pub const FTS_SCHEMA: &str = r#" +CREATE VIRTUAL TABLE IF NOT EXISTS package_search + USING fts5(stem, publisher, summary, description, + content='', tokenize='unicode61'); +"#; + +/// Schema for installed.db - tracks installed packages with manifest blobs. +pub const INSTALLED_SCHEMA: &str = r#" +CREATE TABLE IF NOT EXISTS installed ( + fmri TEXT NOT NULL PRIMARY KEY, + manifest BLOB NOT NULL +); +"#; + +/// Schema for index.db (repository/obsoleted.rs) - server-side obsoleted package index. +pub const OBSOLETED_INDEX_SCHEMA: &str = r#" +CREATE TABLE IF NOT EXISTS obsoleted_packages ( + fmri TEXT NOT NULL PRIMARY KEY, + publisher TEXT NOT NULL, + stem TEXT NOT NULL, + version TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'obsolete', + obsolescence_date TEXT NOT NULL, + deprecation_message TEXT, + obsoleted_by TEXT, + metadata_version INTEGER NOT NULL DEFAULT 1, + content_hash TEXT NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_obsidx_stem ON obsoleted_packages(stem); + +CREATE TABLE IF NOT EXISTS obsoleted_manifests ( + content_hash TEXT NOT NULL PRIMARY KEY, + manifest TEXT NOT NULL +); +"#; + +#[derive(Debug, Error, Diagnostic)] +#[error("Shard building error: {message}")] +#[diagnostic(code(ips::shard_build_error))] +pub struct ShardBuildError { + pub message: String, +} + +impl ShardBuildError { + fn new(msg: impl Into) -> Self { + Self { + message: msg.into(), + } + } +} + +impl From for ShardBuildError { + fn from(e: rusqlite::Error) -> Self { + Self::new(format!("SQLite error: {}", e)) + } +} + +impl From for ShardBuildError { + fn from(e: std::io::Error) -> Self { + Self::new(format!("IO error: {}", e)) + } +} + +impl From for ShardBuildError { + fn from(e: serde_json::Error) -> Self { + Self::new(format!("JSON error: {}", e)) + } +} + +/// Shard metadata entry in catalog.attrs. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ShardEntry { + pub sha256: String, + pub size: u64, + #[serde(rename = "last-modified")] + pub last_modified: String, +} + +/// Shard index JSON structure for catalog/2/catalog.attrs. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ShardIndex { + pub version: u32, + pub created: String, + #[serde(rename = "last-modified")] + pub last_modified: String, + #[serde(rename = "package-count")] + pub package_count: usize, + #[serde(rename = "package-version-count")] + pub package_version_count: usize, + pub shards: BTreeMap, +} + +/// Build catalog shards from JSON catalog parts. +/// +/// Reads catalog parts from `catalog_parts_dir`, generates active.db, obsolete.db, +/// and fts.db, writes them to `output_dir`, and creates catalog.attrs index. +pub fn build_shards( + catalog_parts_dir: &Path, + publisher: &str, + output_dir: &Path, +) -> Result<(), ShardBuildError> { + // Create temp directory for shard generation + fs::create_dir_all(output_dir)?; + let temp_dir = output_dir.join(".tmp"); + fs::create_dir_all(&temp_dir)?; + + // Create shard databases + let active_path = temp_dir.join("active.db"); + let obsolete_path = temp_dir.join("obsolete.db"); + let fts_path = temp_dir.join("fts.db"); + + let mut active_conn = Connection::open(&active_path)?; + let mut obsolete_conn = Connection::open(&obsolete_path)?; + let mut fts_conn = Connection::open(&fts_path)?; + + // Execute schemas + active_conn.execute_batch(ACTIVE_SCHEMA)?; + obsolete_conn.execute_batch(OBSOLETE_SCHEMA)?; + fts_conn.execute_batch(FTS_SCHEMA)?; + + // Read catalog parts + let catalog_manager = CatalogManager::new(catalog_parts_dir, publisher) + .map_err(|e| ShardBuildError::new(format!("Failed to create catalog manager: {}", e)))?; + let mut package_count = 0usize; + let mut package_version_count = 0usize; + + // Begin transactions for batch inserts + let active_tx = active_conn.transaction()?; + let obsolete_tx = obsolete_conn.transaction()?; + let fts_tx = fts_conn.transaction()?; + + { + let mut insert_pkg = active_tx.prepare( + "INSERT OR REPLACE INTO packages (stem, version, publisher) VALUES (?1, ?2, ?3)", + )?; + let mut insert_dep = active_tx.prepare( + "INSERT OR REPLACE INTO dependencies (pkg_stem, pkg_version, pkg_publisher, dep_type, dep_stem, dep_version) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + )?; + let mut insert_obs = obsolete_tx.prepare( + "INSERT OR REPLACE INTO obsolete_packages (publisher, stem, version) VALUES (?1, ?2, ?3)", + )?; + let mut insert_fts = fts_tx.prepare( + "INSERT INTO package_search (stem, publisher, summary, description) VALUES (?1, ?2, ?3, ?4)", + )?; + + // Iterate catalog parts + let part_names: Vec = catalog_manager.attrs().parts.keys().cloned().collect(); + for part_name in part_names { + let part_path = catalog_parts_dir.join(&part_name); + + // Load the CatalogPart + let part = crate::repository::catalog::CatalogPart::load(&part_path) + .map_err(|e| ShardBuildError::new(format!("Failed to load catalog part: {}", e)))?; + + // Iterate through publishers in the catalog part + for (part_publisher, stems) in &part.packages { + // Only process packages for the requested publisher + if part_publisher != publisher { + continue; + } + + // Iterate through package stems + for (pkg_name, versions) in stems { + // Iterate through versions + for version_entry in versions { + let pkg_version = &version_entry.version; + + // Build a minimal manifest from the actions + let mut manifest = Manifest::new(); + + // Parse actions if available + if let Some(actions) = &version_entry.actions { + for action_str in actions { + if action_str.starts_with("set ") { + // Parse "set name=key value=val" format + let parts: Vec<&str> = action_str.split_whitespace().collect(); + if parts.len() >= 3 { + if let Some(name_part) = parts.get(1) { + if let Some(key) = name_part.strip_prefix("name=") { + if let Some(value_part) = parts.get(2) { + if let Some(mut value) = value_part.strip_prefix("value=") { + // Remove quotes + value = value.trim_matches('"'); + + let mut attr = crate::actions::Attr::default(); + attr.key = key.to_string(); + attr.values = vec![value.to_string()]; + manifest.attributes.push(attr); + } + } + } + } + } + } else if action_str.starts_with("depend ") { + // Parse "depend fmri=... type=..." format + let mut dep = crate::actions::Dependency::default(); + for part in action_str.split_whitespace().skip(1) { + if let Some((k, v)) = part.split_once('=') { + match k { + "fmri" => { + if let Ok(f) = crate::fmri::Fmri::parse(v) { + dep.fmri = Some(f); + } + } + "type" => { + dep.dependency_type = v.to_string(); + } + _ => {} + } + } + } + if dep.fmri.is_some() && !dep.dependency_type.is_empty() { + manifest.dependencies.push(dep); + } + } + } + } + + // Determine if obsolete + let is_obsolete = crate::image::catalog::is_package_obsolete(&manifest); + + // Count all package versions + package_version_count += 1; + + // Obsolete packages go only to obsolete.db, non-obsolete go to active.db + if is_obsolete { + insert_obs.execute(rusqlite::params![publisher, pkg_name, pkg_version])?; + } else { + // Insert into packages table (active.db) + insert_pkg.execute(rusqlite::params![pkg_name, pkg_version, publisher])?; + + // Extract and insert dependencies + for dep in &manifest.dependencies { + if dep.dependency_type == "require" || dep.dependency_type == "incorporate" { + if let Some(dep_fmri) = &dep.fmri { + let dep_stem = dep_fmri.stem(); + let dep_version = dep_fmri.version.as_ref().map(|v| v.to_string()); + insert_dep.execute(rusqlite::params![ + pkg_name, + pkg_version, + publisher, + &dep.dependency_type, + dep_stem, + dep_version + ])?; + } + } + } + } + + // Extract summary and description for FTS + let summary = manifest + .attributes + .iter() + .find(|a| a.key == "pkg.summary") + .and_then(|a| a.values.first()) + .map(|s| s.as_str()) + .unwrap_or(""); + let description = manifest + .attributes + .iter() + .find(|a| a.key == "pkg.description") + .and_then(|a| a.values.first()) + .map(|s| s.as_str()) + .unwrap_or(""); + + insert_fts.execute(rusqlite::params![ + pkg_name, + publisher, + summary, + description + ])?; + } + } + } + } + } + + // Commit transactions + active_tx.commit()?; + obsolete_tx.commit()?; + fts_tx.commit()?; + + // Count unique packages (stems) + let count: i64 = active_conn.query_row( + "SELECT COUNT(DISTINCT stem) FROM packages", + [], + |row| row.get(0), + )?; + package_count = count as usize; + + // Close connections + drop(active_conn); + drop(obsolete_conn); + drop(fts_conn); + + // Compute SHA-256 hashes and build index + let mut shards = BTreeMap::new(); + let now = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string(); + + for (name, path) in [ + ("active.db", &active_path), + ("obsolete.db", &obsolete_path), + ("fts.db", &fts_path), + ] { + let bytes = fs::read(path)?; + let mut hasher = Sha256::new(); + hasher.update(&bytes); + let hash = format!("{:x}", hasher.finalize()); + let size = bytes.len() as u64; + + shards.insert( + name.to_string(), + ShardEntry { + sha256: hash.clone(), + size, + last_modified: now.clone(), + }, + ); + + // Copy shard to output directory with both original name and hash-based name + // Keep original name for client-side use (e.g., active.db, obsolete.db) + let named_path = output_dir.join(name); + fs::copy(path, &named_path)?; + + // Also copy to hash-based name for content-addressed server distribution + let hash_path = output_dir.join(&hash); + fs::copy(path, &hash_path)?; + } + + // Write catalog.attrs + let index = ShardIndex { + version: 2, + created: now.clone(), + last_modified: now, + package_count, + package_version_count, + shards, + }; + let index_json = serde_json::to_string_pretty(&index)?; + fs::write(output_dir.join("catalog.attrs"), index_json)?; + + // Clean up temp directory + fs::remove_dir_all(&temp_dir).ok(); + + Ok(()) +} + +/// Helper function for tests: populate active.db with a single package. +/// Creates tables if absent (idempotent). +pub fn populate_active_db( + db_path: &Path, + fmri: &Fmri, + manifest: &Manifest, +) -> Result<(), ShardBuildError> { + let mut conn = Connection::open(db_path)?; + conn.execute_batch(ACTIVE_SCHEMA)?; + + let tx = conn.transaction()?; + { + tx.execute( + "INSERT OR REPLACE INTO packages (stem, version, publisher) VALUES (?1, ?2, ?3)", + rusqlite::params![ + fmri.stem(), + fmri.version(), + fmri.publisher.as_deref().unwrap_or("") + ], + )?; + + for dep in &manifest.dependencies { + if dep.dependency_type == "require" || dep.dependency_type == "incorporate" { + if let Some(dep_fmri) = &dep.fmri { + let dep_stem = dep_fmri.stem(); + let dep_version = dep_fmri.version.as_ref().map(|v| v.to_string()); + tx.execute( + "INSERT OR REPLACE INTO dependencies (pkg_stem, pkg_version, pkg_publisher, dep_type, dep_stem, dep_version) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + rusqlite::params![ + fmri.stem(), + fmri.version(), + fmri.publisher.as_deref().unwrap_or(""), + &dep.dependency_type, + dep_stem, + dep_version + ], + )?; + } + } + } + } + tx.commit()?; + Ok(()) +} + +/// Helper function for tests: mark a package as obsolete in obsolete.db. +/// Creates tables if absent (idempotent). +pub fn populate_obsolete_db(db_path: &Path, fmri: &Fmri) -> Result<(), ShardBuildError> { + let mut conn = Connection::open(db_path)?; + conn.execute_batch(OBSOLETE_SCHEMA)?; + + conn.execute( + "INSERT OR REPLACE INTO obsolete_packages (publisher, stem, version) VALUES (?1, ?2, ?3)", + rusqlite::params![ + fmri.publisher.as_deref().unwrap_or(""), + fmri.stem(), + fmri.version() + ], + )?; + Ok(()) +} + +// Note: compress_json_lz4, decode_manifest_bytes, and is_package_obsolete +// are available as pub(crate) in crate::image::catalog and can be used +// within libips but not re-exported. diff --git a/libips/src/solver/mod.rs b/libips/src/solver/mod.rs index 7570921..52d71e8 100644 --- a/libips/src/solver/mod.rs +++ b/libips/src/solver/mod.rs @@ -18,55 +18,23 @@ use miette::Diagnostic; // Begin resolvo wiring imports (names discovered by compiler) // We start broad and refine with compiler guidance. -use lz4::Decoder as Lz4Decoder; -use redb::{ReadableDatabase, ReadableTable}; use resolvo::{ self, Candidates, Condition, ConditionId, ConditionalRequirement, Dependencies as RDependencies, DependencyProvider, HintDependenciesAvailable, Interner, KnownDependencies, Mapping, NameId, Problem as RProblem, SolvableId, Solver as RSolver, SolverCache, StringId, UnsolvableOrCancelled, VersionSetId, VersionSetUnionId, }; +use rusqlite::{Connection, OpenFlags}; use std::cell::RefCell; use std::collections::{BTreeMap, HashMap}; use std::fmt::Display; -use std::io::{Cursor, Read}; use thiserror::Error; use crate::actions::Manifest; -use crate::image::catalog::{CATALOG_TABLE, INCORPORATE_TABLE}; // Public advice API lives in a sibling module pub mod advice; -// Local helpers to decode manifest bytes stored in catalog DB (JSON or LZ4-compressed JSON) -fn is_likely_json_local(bytes: &[u8]) -> bool { - let mut i = 0; - while i < bytes.len() && matches!(bytes[i], b' ' | b'\n' | b'\r' | b'\t') { - i += 1; - } - if i >= bytes.len() { - return false; - } - matches!(bytes[i], b'{' | b'[') -} - -fn decode_manifest_bytes_local(bytes: &[u8]) -> Result { - if is_likely_json_local(bytes) { - return serde_json::from_slice::(bytes); - } - // Try LZ4; on failure, fall back to JSON attempt - if let Ok(mut dec) = Lz4Decoder::new(Cursor::new(bytes)) { - let mut out = Vec::new(); - if dec.read_to_end(&mut out).is_ok() { - if let Ok(m) = serde_json::from_slice::(&out) { - return Ok(m); - } - } - } - // Fallback to JSON parse of original bytes - serde_json::from_slice::(bytes) -} - #[derive(Clone, Debug)] struct PkgCand { #[allow(dead_code)] @@ -85,11 +53,8 @@ enum VersionSetKind { struct IpsProvider<'a> { image: &'a Image, - // Persistent database handles and read transactions for catalog/obsoleted - _catalog_db: redb::Database, - catalog_tx: redb::ReadTransaction, - _obsoleted_db: redb::Database, - _obsoleted_tx: redb::ReadTransaction, + // SQLite connection to active.db (catalog), opened read-only + catalog_conn: Connection, // interner storages names: Mapping, name_by_str: BTreeMap, @@ -108,24 +73,21 @@ use crate::image::Image; impl<'a> IpsProvider<'a> { fn new(image: &'a Image) -> Result { - // Open databases and keep read transactions alive for the provider lifetime - let catalog_db = redb::Database::open(image.catalog_db_path()) - .map_err(|e| SolverError::new(format!("open catalog db: {}", e)))?; - let catalog_tx = catalog_db - .begin_read() - .map_err(|e| SolverError::new(format!("begin read catalog db: {}", e)))?; - let obsoleted_db = redb::Database::open(image.obsoleted_db_path()) - .map_err(|e| SolverError::new(format!("open obsoleted db: {}", e)))?; - let obsoleted_tx = obsoleted_db - .begin_read() - .map_err(|e| SolverError::new(format!("begin read obsoleted db: {}", e)))?; + // Open active.db (catalog) read-only with WAL mode for better concurrency + let catalog_conn = Connection::open_with_flags( + &image.active_db_path(), + OpenFlags::SQLITE_OPEN_READ_ONLY, + ) + .map_err(|e| SolverError::new(format!("open catalog db: {}", e)))?; + + // Enable WAL mode for better concurrency (ignored if already set) + catalog_conn + .pragma_update(None, "journal_mode", "WAL") + .ok(); let mut prov = IpsProvider { image, - _catalog_db: catalog_db, - catalog_tx, - _obsoleted_db: obsoleted_db, - _obsoleted_tx: obsoleted_tx, + catalog_conn, names: Mapping::default(), name_by_str: BTreeMap::new(), strings: Mapping::default(), @@ -141,80 +103,70 @@ impl<'a> IpsProvider<'a> { } fn build_index(&mut self) -> Result<(), SolverError> { - use crate::image::catalog::CATALOG_TABLE; - // Iterate catalog table and build in-memory index of non-obsolete candidates - let table = self - .catalog_tx - .open_table(CATALOG_TABLE) - .map_err(|e| SolverError::new(format!("open catalog table: {}", e)))?; + // Query packages table directly - no manifest decoding needed + // Use a scope to ensure stmt is dropped before we start mutating self + let collected_rows: Vec> = { + let mut stmt = self + .catalog_conn + .prepare("SELECT stem, version, publisher FROM packages") + .map_err(|e| SolverError::new(format!("prepare packages query: {}", e)))?; + + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + }) + .map_err(|e| SolverError::new(format!("query packages: {}", e)))?; + + // Collect all rows into a Vec to avoid holding a borrow on the connection + rows.collect() + }; // Temporary map: stem string -> Vec let mut by_stem: BTreeMap> = BTreeMap::new(); - for entry in table - .iter() - .map_err(|e| SolverError::new(format!("iterate catalog table: {}", e)))? - { - let (k, v) = - entry.map_err(|e| SolverError::new(format!("read catalog entry: {}", e)))?; - let key = k.value(); // stem@version - // Try to decode manifest and extract full FMRI (including publisher) - let mut pushed = false; - if let Ok(manifest) = decode_manifest_bytes_local(v.value()) { - if let Some(attr) = manifest.attributes.iter().find(|a| a.key == "pkg.fmri") { - if let Some(fmri_str) = attr.values.first() { - if let Ok(mut fmri) = Fmri::parse(fmri_str) { - // Ensure publisher is present; if missing/empty, use image default publisher - let missing_pub = fmri - .publisher - .as_deref() - .map(|s| s.is_empty()) - .unwrap_or(true); - if missing_pub { - if let Ok(defp) = self.image.default_publisher() { - fmri.publisher = Some(defp.name.clone()); - } - } - by_stem - .entry(fmri.stem().to_string()) - .or_default() - .push(fmri); - pushed = true; - } - } + for row_result in collected_rows { + let (stem, version, publisher) = row_result + .map_err(|e| SolverError::new(format!("read package row: {}", e)))?; + + // Parse version + let ver_obj = crate::fmri::Version::parse(&version).ok(); + + // Build FMRI with publisher, stem, and version + let mut fmri = if let Some(v) = ver_obj { + Fmri::with_publisher(&publisher, &stem, Some(v)) + } else { + // No parsable version; still record a minimal FMRI without version + Fmri::with_publisher(&publisher, &stem, None) + }; + + // Normalize: empty publisher string -> None + if fmri.publisher.as_deref() == Some("") { + fmri.publisher = None; + } + + // If publisher is still None/empty, try using image default publisher + let missing_pub = fmri + .publisher + .as_deref() + .map(|s| s.is_empty()) + .unwrap_or(true); + if missing_pub { + if let Ok(defp) = self.image.default_publisher() { + fmri.publisher = Some(defp.name.clone()); } } - // Fallback: derive FMRI from catalog key if we couldn't push from manifest - if !pushed { - if let Some((stem, ver_str)) = key.split_once('@') { - let ver_obj = crate::fmri::Version::parse(ver_str).ok(); - // Prefer default publisher if configured; else leave None by constructing and then setting publisher - let mut fmri = if let Some(v) = ver_obj.clone() { - if let Ok(defp) = self.image.default_publisher() { - Fmri::with_publisher(&defp.name, stem, Some(v)) - } else { - Fmri::with_version(stem, v) - } - } else { - // No parsable version; still record a minimal FMRI without version - if let Ok(defp) = self.image.default_publisher() { - Fmri::with_publisher(&defp.name, stem, None) - } else { - Fmri::with_publisher("", stem, None) - } - }; - // Normalize: empty publisher string -> None - if fmri.publisher.as_deref() == Some("") { - fmri.publisher = None; - } - by_stem.entry(stem.to_string()).or_default().push(fmri); - } - } + by_stem.entry(stem).or_default().push(fmri); } // Intern and populate solvables per stem - for (stem, mut fmris) in by_stem { + // Collect into Vec to avoid borrow checker issues with mutating self while iterating + let stems_and_fmris: Vec<(String, Vec)> = by_stem.into_iter().collect(); + for (stem, mut fmris) in stems_and_fmris { let name_id = self.intern_name(&stem); // Sort fmris newest-first using IPS ordering fmris.sort_by(|a, b| version_order_desc(a, b)); @@ -254,22 +206,13 @@ impl<'a> IpsProvider<'a> { } fn lookup_incorporated_release(&self, stem: &str) -> Option { - if let Ok(table) = self.catalog_tx.open_table(INCORPORATE_TABLE) { - if let Ok(Some(rel)) = table.get(stem) { - return Some(String::from_utf8_lossy(rel.value()).to_string()); - } - } - None - } - - fn read_manifest_from_catalog(&self, fmri: &Fmri) -> Option { - let key = format!("{}@{}", fmri.stem(), fmri.version()); - if let Ok(table) = self.catalog_tx.open_table(CATALOG_TABLE) { - if let Ok(Some(bytes)) = table.get(key.as_str()) { - return decode_manifest_bytes_local(bytes.value()).ok(); - } - } - None + self.catalog_conn + .query_row( + "SELECT release FROM incorporate_locks WHERE stem = ?1", + rusqlite::params![stem], + |row| row.get(0), + ) + .ok() } } @@ -516,9 +459,31 @@ impl<'a> DependencyProvider for IpsProvider<'a> { async fn get_dependencies(&self, solvable: SolvableId) -> RDependencies { let pkg = self.solvables.get(solvable).unwrap(); let fmri = &pkg.fmri; - let manifest_opt = self.read_manifest_from_catalog(fmri); - let Some(manifest) = manifest_opt else { - return RDependencies::Known(KnownDependencies::default()); + + // Query dependencies table directly instead of decoding manifest + let mut stmt = match self.catalog_conn.prepare( + "SELECT dep_stem, dep_version FROM dependencies + WHERE pkg_stem = ?1 AND pkg_version = ?2 AND pkg_publisher = ?3 AND dep_type = 'require'" + ) { + Ok(s) => s, + Err(_) => return RDependencies::Known(KnownDependencies::default()), + }; + + let parent_stem = fmri.stem(); + let parent_version = fmri.version(); + let parent_publisher = fmri.publisher.as_deref().unwrap_or(""); + + let rows = match stmt.query_map( + rusqlite::params![parent_stem, parent_version, parent_publisher], + |row| { + Ok(( + row.get::<_, String>(0)?, // dep_stem + row.get::<_, Option>(1)?, // dep_version + )) + }, + ) { + Ok(r) => r, + Err(_) => return RDependencies::Known(KnownDependencies::default()), }; // Build requirements for "require" deps @@ -526,38 +491,43 @@ impl<'a> DependencyProvider for IpsProvider<'a> { let parent_branch = fmri.version.as_ref().and_then(|v| v.branch.clone()); let parent_pub = fmri.publisher.as_deref(); - for d in manifest - .dependencies - .iter() - .filter(|d| d.dependency_type == "require") - { - if let Some(df) = &d.fmri { - let stem = df.stem().to_string(); - let Some(child_name_id) = self.name_by_str.get(&stem).copied() else { - // If the dependency name isn't present in the catalog index, skip it - continue; - }; - // Create version set by release (from dep expr) and branch (from parent) - let vs_kind = match (&df.version, &parent_branch) { - (Some(ver), Some(branch)) => VersionSetKind::ReleaseAndBranch { - release: ver.release.clone(), - branch: branch.clone(), - }, - (Some(ver), None) => VersionSetKind::ReleaseEq(ver.release.clone()), - (None, Some(branch)) => VersionSetKind::BranchEq(branch.clone()), - (None, None) => VersionSetKind::Any, - }; - let vs_id = self.version_set_for(child_name_id, vs_kind); - reqs.push(ConditionalRequirement::from(vs_id)); + for row_result in rows { + let (dep_stem, dep_version_str) = match row_result { + Ok(r) => r, + Err(_) => continue, + }; - // Set publisher preferences for the child to parent-first, then image order - let order = build_publisher_preference(parent_pub, self.image); - self.publisher_prefs - .borrow_mut() - .entry(child_name_id) - .or_insert(order); - } + let Some(child_name_id) = self.name_by_str.get(&dep_stem).copied() else { + // If the dependency name isn't present in the catalog index, skip it + continue; + }; + + // Parse dep_version to extract release component + let dep_version = dep_version_str + .as_ref() + .and_then(|s| crate::fmri::Version::parse(s).ok()); + + // Create version set by release (from dep expr) and branch (from parent) + let vs_kind = match (&dep_version, &parent_branch) { + (Some(ver), Some(branch)) => VersionSetKind::ReleaseAndBranch { + release: ver.release.clone(), + branch: branch.clone(), + }, + (Some(ver), None) => VersionSetKind::ReleaseEq(ver.release.clone()), + (None, Some(branch)) => VersionSetKind::BranchEq(branch.clone()), + (None, None) => VersionSetKind::Any, + }; + let vs_id = self.version_set_for(child_name_id, vs_kind); + reqs.push(ConditionalRequirement::from(vs_id)); + + // Set publisher preferences for the child to parent-first, then image order + let order = build_publisher_preference(parent_pub, self.image); + self.publisher_prefs + .borrow_mut() + .entry(child_name_id) + .or_insert(order); } + RDependencies::Known(KnownDependencies { requirements: reqs, constrains: vec![], @@ -820,18 +790,6 @@ pub fn resolve_install( } name_to_fmris.insert(*name_id, v); } - // Snapshot: Catalog manifest cache keyed by stem@version for all candidates - let mut key_to_manifest: HashMap = HashMap::new(); - for fmris in name_to_fmris.values() { - for fmri in fmris { - let key = format!("{}@{}", fmri.stem(), fmri.version()); - if !key_to_manifest.contains_key(&key) { - if let Some(man) = provider.read_manifest_from_catalog(fmri) { - key_to_manifest.insert(key, man); - } - } - } - } // Run the solver let roots_for_err: Vec = root_names.iter().map(|(_, c)| c.clone()).collect(); @@ -864,25 +822,18 @@ pub fn resolve_install( let mut plan = InstallPlan::default(); for sid in solution_ids { if let Some(fmri) = sid_to_fmri.get(&sid).cloned() { - // Prefer repository manifest; fallback to preloaded catalog snapshot, then image catalog - let key = format!("{}@{}", fmri.stem(), fmri.version()); + // Fetch manifest from repository or catalog cache let manifest = match image_ref.get_manifest_from_repository(&fmri) { Ok(m) => m, - Err(repo_err) => { - if let Some(m) = key_to_manifest.get(&key).cloned() { - m - } else { - match image_ref.get_manifest_from_catalog(&fmri) { - Ok(Some(m)) => m, - _ => { - return Err(SolverError::new(format!( - "failed to obtain manifest for {}: {}", - fmri, repo_err - ))); - } - } + Err(repo_err) => match image_ref.get_manifest_from_catalog(&fmri) { + Ok(Some(m)) => m, + _ => { + return Err(SolverError::new(format!( + "failed to obtain manifest for {}: {}", + fmri, repo_err + ))); } - } + }, }; plan.reasons.push(format!("selected {} via solver", fmri)); plan.add.push(ResolvedPkg { fmri, manifest }); @@ -933,8 +884,7 @@ mod solver_integration_tests { use crate::actions::Dependency; use crate::fmri::Version; use crate::image::ImageType; - use crate::image::catalog::{CATALOG_TABLE, OBSOLETED_TABLE}; - use redb::Database; + use crate::repository::sqlite_catalog; use tempfile::tempdir; fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version { @@ -970,34 +920,13 @@ mod solver_integration_tests { } fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) { - let db = Database::open(image.catalog_db_path()).expect("open catalog db"); - let tx = db.begin_write().expect("begin write"); - { - let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table"); - let key = format!("{}@{}", fmri.stem(), fmri.version()); - let val = serde_json::to_vec(manifest).expect("serialize manifest"); - table - .insert(key.as_str(), val.as_slice()) - .expect("insert manifest"); - } - tx.commit().expect("commit"); + sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest) + .expect("populate active db"); } fn mark_obsolete(image: &Image, fmri: &Fmri) { - let db = Database::open(image.obsoleted_db_path()).expect("open obsoleted db"); - let tx = db.begin_write().expect("begin write"); - { - let mut table = tx - .open_table(OBSOLETED_TABLE) - .expect("open obsoleted table"); - let key = fmri.to_string(); - // store empty value - let empty: Vec = Vec::new(); - table - .insert(key.as_str(), empty.as_slice()) - .expect("insert obsolete"); - } - tx.commit().expect("commit"); + sqlite_catalog::populate_obsolete_db(&image.obsolete_db_path(), fmri) + .expect("populate obsolete db"); } fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image { @@ -1322,8 +1251,7 @@ mod solver_error_message_tests { use crate::actions::{Dependency, Manifest}; use crate::fmri::{Fmri, Version}; use crate::image::ImageType; - use crate::image::catalog::CATALOG_TABLE; - use redb::Database; + use crate::repository::sqlite_catalog; fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version { let mut v = Version::new(release); @@ -1354,17 +1282,8 @@ mod solver_error_message_tests { } fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) { - let db = Database::open(image.catalog_db_path()).expect("open catalog db"); - let tx = db.begin_write().expect("begin write"); - { - let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table"); - let key = format!("{}@{}", fmri.stem(), fmri.version()); - let val = serde_json::to_vec(manifest).expect("serialize manifest"); - table - .insert(key.as_str(), val.as_slice()) - .expect("insert manifest"); - } - tx.commit().expect("commit"); + sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest) + .expect("populate active db"); } #[test] @@ -1427,8 +1346,7 @@ mod incorporate_lock_tests { use crate::actions::Dependency; use crate::fmri::Version; use crate::image::ImageType; - use crate::image::catalog::CATALOG_TABLE; - use redb::Database; + use crate::repository::sqlite_catalog; use tempfile::tempdir; fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version { @@ -1447,17 +1365,8 @@ mod incorporate_lock_tests { } fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) { - let db = Database::open(image.catalog_db_path()).expect("open catalog db"); - let tx = db.begin_write().expect("begin write"); - { - let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table"); - let key = format!("{}@{}", fmri.stem(), fmri.version()); - let val = serde_json::to_vec(manifest).expect("serialize manifest"); - table - .insert(key.as_str(), val.as_slice()) - .expect("insert manifest"); - } - tx.commit().expect("commit"); + sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest) + .expect("populate active db"); } fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image { @@ -1638,8 +1547,7 @@ mod composite_release_tests { use crate::actions::{Dependency, Manifest}; use crate::fmri::{Fmri, Version}; use crate::image::ImageType; - use crate::image::catalog::CATALOG_TABLE; - use redb::Database; + use crate::repository::sqlite_catalog; fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version { let mut v = Version::new(release); @@ -1657,17 +1565,8 @@ mod composite_release_tests { } fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) { - let db = Database::open(image.catalog_db_path()).expect("open catalog db"); - let tx = db.begin_write().expect("begin write"); - { - let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table"); - let key = format!("{}@{}", fmri.stem(), fmri.version()); - let val = serde_json::to_vec(manifest).expect("serialize manifest"); - table - .insert(key.as_str(), val.as_slice()) - .expect("insert manifest"); - } - tx.commit().expect("commit"); + sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest) + .expect("populate active db"); } fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image { @@ -1772,8 +1671,7 @@ mod circular_dependency_tests { use crate::actions::Dependency; use crate::fmri::{Fmri, Version}; use crate::image::ImageType; - use crate::image::catalog::CATALOG_TABLE; - use redb::Database; + use crate::repository::sqlite_catalog; use std::collections::HashSet; fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version { @@ -1809,17 +1707,8 @@ mod circular_dependency_tests { } fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) { - let db = Database::open(image.catalog_db_path()).expect("open catalog db"); - let tx = db.begin_write().expect("begin write"); - { - let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table"); - let key = format!("{}@{}", fmri.stem(), fmri.version()); - let val = serde_json::to_vec(manifest).expect("serialize manifest"); - table - .insert(key.as_str(), val.as_slice()) - .expect("insert manifest"); - } - tx.commit().expect("commit"); + sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest) + .expect("populate active db"); } fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image { diff --git a/pkg6depotd/Cargo.toml b/pkg6depotd/Cargo.toml index d932f06..92ad62e 100644 --- a/pkg6depotd/Cargo.toml +++ b/pkg6depotd/Cargo.toml @@ -31,6 +31,8 @@ serde_json = "1.0" dirs = "6" nix = { version = "0.30", features = ["signal", "process", "user", "fs"] } sha1 = "0.10" +sha2 = "0.10" +rusqlite = { version = "0.31", default-features = false } chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } flate2 = "1" httpdate = "1" @@ -55,3 +57,7 @@ reqwest = { version = "0.12", features = ["blocking", "json"] } assert_cmd = "2" predicates = "3" tempfile = "3" + +[features] +default = ["bundled-sqlite"] +bundled-sqlite = ["rusqlite/bundled"] diff --git a/pkg6depotd/src/http/handlers/mod.rs b/pkg6depotd/src/http/handlers/mod.rs index ff5c198..84de7b0 100644 --- a/pkg6depotd/src/http/handlers/mod.rs +++ b/pkg6depotd/src/http/handlers/mod.rs @@ -4,4 +4,5 @@ pub mod info; pub mod manifest; pub mod publisher; pub mod search; +pub mod shard; pub mod versions; diff --git a/pkg6depotd/src/http/handlers/shard.rs b/pkg6depotd/src/http/handlers/shard.rs new file mode 100644 index 0000000..b47e868 --- /dev/null +++ b/pkg6depotd/src/http/handlers/shard.rs @@ -0,0 +1,127 @@ +use crate::errors::DepotError; +use crate::repo::DepotRepo; +use axum::extract::{Path, Request, State}; +use axum::http::header; +use axum::response::{IntoResponse, Response}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::fs; +use std::sync::Arc; +use tower::ServiceExt; +use tower_http::services::ServeFile; + +/// Shard metadata entry in catalog.attrs. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ShardEntry { + sha256: String, + size: u64, + #[serde(rename = "last-modified")] + last_modified: String, +} + +/// Shard index JSON structure for catalog/2/catalog.attrs. +#[derive(Debug, Clone, Serialize, Deserialize)] +struct ShardIndex { + version: u32, + created: String, + #[serde(rename = "last-modified")] + last_modified: String, + #[serde(rename = "package-count")] + package_count: usize, + #[serde(rename = "package-version-count")] + package_version_count: usize, + shards: BTreeMap, +} + +/// GET /{publisher}/catalog/2/catalog.attrs +pub async fn get_shard_index( + State(repo): State>, + Path(publisher): Path, +) -> Result { + let shard_dir = repo.shard_dir(&publisher); + let index_path = shard_dir.join("catalog.attrs"); + + if !index_path.exists() { + return Err(DepotError::Repo( + libips::repository::RepositoryError::NotFound( + "catalog.attrs not found - shards not yet built".to_string(), + ), + )); + } + + let content = fs::read_to_string(&index_path) + .map_err(|e| DepotError::Server(format!("Failed to read catalog.attrs: {}", e)))?; + + Ok(([(header::CONTENT_TYPE, "application/json")], content).into_response()) +} + +/// GET /{publisher}/catalog/2/{sha256} +pub async fn get_shard_blob( + State(repo): State>, + Path((publisher, sha256)): Path<(String, String)>, + req: Request, +) -> Result { + let shard_dir = repo.shard_dir(&publisher); + let index_path = shard_dir.join("catalog.attrs"); + + if !index_path.exists() { + return Err(DepotError::Repo( + libips::repository::RepositoryError::NotFound( + "catalog.attrs not found - shards not yet built".to_string(), + ), + )); + } + + // Read index to validate hash + let index_content = fs::read_to_string(&index_path) + .map_err(|e| DepotError::Server(format!("Failed to read catalog.attrs: {}", e)))?; + let index: ShardIndex = serde_json::from_str(&index_content) + .map_err(|e| DepotError::Server(format!("Failed to parse catalog.attrs: {}", e)))?; + + // Find which shard file corresponds to this hash + let mut shard_path: Option = None; + for (name, entry) in &index.shards { + if entry.sha256 == sha256 { + shard_path = Some(shard_dir.join(&sha256)); + break; + } + } + + let Some(path) = shard_path else { + return Err(DepotError::Repo( + libips::repository::RepositoryError::NotFound(format!( + "Shard with hash {} not found", + sha256 + )), + )); + }; + + if !path.exists() { + return Err(DepotError::Repo( + libips::repository::RepositoryError::NotFound(format!( + "Shard file {} not found on disk", + sha256 + )), + )); + } + + // Serve the file + let service = ServeFile::new(path); + let result = service.oneshot(req).await; + + match result { + Ok(mut res) => { + // Add cache headers - content is content-addressed and immutable + res.headers_mut().insert( + header::CONTENT_TYPE, + header::HeaderValue::from_static("application/octet-stream"), + ); + res.headers_mut().insert( + header::CACHE_CONTROL, + header::HeaderValue::from_static("public, immutable, max-age=86400"), + ); + Ok(res.into_response()) + } + Err(e) => Err(DepotError::Server(e.to_string())), + } +} diff --git a/pkg6depotd/src/http/handlers/versions.rs b/pkg6depotd/src/http/handlers/versions.rs index 83db5bf..c097442 100644 --- a/pkg6depotd/src/http/handlers/versions.rs +++ b/pkg6depotd/src/http/handlers/versions.rs @@ -68,7 +68,7 @@ pub async fn get_versions() -> impl IntoResponse { }, SupportedOperation { op: Operation::Catalog, - versions: vec![1], + versions: vec![1, 2], }, SupportedOperation { op: Operation::Manifest, diff --git a/pkg6depotd/src/http/routes.rs b/pkg6depotd/src/http/routes.rs index bb2f97f..22047c5 100644 --- a/pkg6depotd/src/http/routes.rs +++ b/pkg6depotd/src/http/routes.rs @@ -1,5 +1,5 @@ use crate::http::admin; -use crate::http::handlers::{catalog, file, info, manifest, publisher, search, versions}; +use crate::http::handlers::{catalog, file, info, manifest, publisher, search, shard, versions}; use crate::repo::DepotRepo; use axum::{ Router, @@ -16,6 +16,14 @@ pub fn app_router(state: Arc) -> Router { "/{publisher}/catalog/1/{filename}", get(catalog::get_catalog_v1).head(catalog::get_catalog_v1), ) + .route( + "/{publisher}/catalog/2/catalog.attrs", + get(shard::get_shard_index).head(shard::get_shard_index), + ) + .route( + "/{publisher}/catalog/2/{sha256}", + get(shard::get_shard_blob).head(shard::get_shard_blob), + ) .route( "/{publisher}/manifest/0/{fmri}", get(manifest::get_manifest).head(manifest::get_manifest), diff --git a/pkg6depotd/src/repo.rs b/pkg6depotd/src/repo.rs index f125cd2..9d0108d 100644 --- a/pkg6depotd/src/repo.rs +++ b/pkg6depotd/src/repo.rs @@ -104,6 +104,13 @@ impl DepotRepo { self.cache_max_age } + pub fn shard_dir(&self, publisher: &str) -> PathBuf { + self.root + .join("publisher") + .join(publisher) + .join("catalog2") + } + pub fn get_catalog_file_path(&self, publisher: &str, filename: &str) -> Result { let backend = self .backend