CloudNebulaProject/ips
1
0
Fork 0
mirror of https://codeberg.org/Toasterson/ips.git synced 2026-04-10 13:20:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Refactor: Replace redb with rusqlite for catalog handling

Browse source 
- Transitioned the catalog backend from `redb` to `rusqlite` for better compatibility and concurrency.
- Updated `IpsProvider` to use SQLite for package querying, dependency resolution, and obsolescence handling.
- Removed `decode_manifest_bytes_local` and unused `manifest` cache logic.
- Simplified catalog-related functions and integrated `sqlite_catalog` module.
- Enhanced test functions and added schemas for managing SQLite databases.
This commit is contained in:
Till Wegmueller 2026-02-04 22:39:42 +01:00
parent 4ab529f4c7
commit def11a1dfb
No known key found for this signature in database
18 changed files with 1790 additions and 2106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

114
Cargo.lock generated
View file

@ -326,26 +326,6 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bincode"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
dependencies = [
"bincode_derive",
"serde",
"unty",
]
[[package]]
name = "bincode_derive"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
dependencies = [
"virtue",
]
[[package]] [[package]]
name = "bitflags" name = "bitflags"
version = "2.10.0" version = "2.10.0"
@ -765,6 +745,18 @@ dependencies = [
"pin-project-lite", "pin-project-lite",
] ]
[[package]]
name = "fallible-iterator"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]] [[package]]
name = "fastrand" name = "fastrand"
version = "2.3.0" version = "2.3.0"
@ -1020,12 +1012,6 @@ dependencies = [
"tracing", "tracing",
] ]
[[package]]
name = "half"
version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403"
[[package]] [[package]]
name = "hashbrown" name = "hashbrown"
version = "0.14.5" version = "0.14.5"
@ -1051,6 +1037,15 @@ version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "hashlink"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
dependencies = [
"hashbrown 0.14.5",
]
[[package]] [[package]]
name = "heck" name = "heck"
version = "0.4.1" version = "0.4.1"
@ -1463,7 +1458,7 @@ checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"
name = "libips" name = "libips"
version = "0.5.3" version = "0.5.3"
dependencies = [ dependencies = [
"bincode", "base64 0.22.1",
"chrono", "chrono",
"diff-struct", "diff-struct",
"flate2", "flate2",
@ -1474,14 +1469,13 @@ dependencies = [
"object", "object",
"pest", "pest",
"pest_derive", "pest_derive",
"redb",
"regex", "regex",
"reqwest", "reqwest",
"resolvo", "resolvo",
"rusqlite",
"rust-ini", "rust-ini",
"semver", "semver",
"serde", "serde",
"serde_cbor",
"serde_json", "serde_json",
"sha1", "sha1",
"sha2", "sha2",
@ -1503,6 +1497,17 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "libsqlite3-sys"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
dependencies = [
"cc",
"pkg-config",
"vcpkg",
]
[[package]] [[package]]
name = "linux-raw-sys" name = "linux-raw-sys"
version = "0.11.0" version = "0.11.0"
@ -2120,10 +2125,12 @@ dependencies = [
"opentelemetry_sdk", "opentelemetry_sdk",
"predicates", "predicates",
"reqwest", "reqwest",
"rusqlite",
"rustls", "rustls",
"serde", "serde",
"serde_json", "serde_json",
"sha1", "sha1",
"sha2",
"socket2", "socket2",
"tempfile", "tempfile",
"thiserror 2.0.17", "thiserror 2.0.17",
@ -2197,7 +2204,7 @@ dependencies = [
"reqwest", "reqwest",
"shellexpand", "shellexpand",
"specfile", "specfile",
"thiserror 1.0.69", "thiserror 2.0.17",
"url", "url",
"which", "which",
] ]
@ -2411,15 +2418,6 @@ dependencies = [
"getrandom 0.3.4", "getrandom 0.3.4",
] ]
[[package]]
name = "redb"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae323eb086579a3769daa2c753bb96deb95993c534711e0dbe881b5192906a06"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.5.18" version = "0.5.18"
@ -2548,6 +2546,20 @@ dependencies = [
"windows-sys 0.52.0", "windows-sys 0.52.0",
] ]
[[package]]
name = "rusqlite"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
dependencies = [
"bitflags",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"smallvec",
]
[[package]] [[package]]
name = "rust-ini" name = "rust-ini"
version = "0.21.3" version = "0.21.3"
@ -2729,16 +2741,6 @@ dependencies = [
"serde_derive", "serde_derive",
] ]
[[package]]
name = "serde_cbor"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
dependencies = [
"half",
"serde",
]
[[package]] [[package]]
name = "serde_core" name = "serde_core"
version = "1.0.228" version = "1.0.228"
@ -2895,7 +2897,7 @@ dependencies = [
"anyhow", "anyhow",
"pest", "pest",
"pest_derive", "pest_derive",
"thiserror 1.0.69", "thiserror 2.0.17",
] ]
[[package]] [[package]]
@ -3479,12 +3481,6 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "unty"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
[[package]] [[package]]
name = "url" name = "url"
version = "2.5.7" version = "2.5.7"
@ -3554,12 +3550,6 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "virtue"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
[[package]] [[package]]
name = "wait-timeout" name = "wait-timeout"
version = "0.2.1" version = "0.2.1"

9
libips/Cargo.toml
View file

@ -32,20 +32,19 @@ pest_derive = "2.1.0"
strum = { version = "0.27", features = ["derive"] } strum = { version = "0.27", features = ["derive"] }
serde = { version = "1.0.207", features = ["derive"] } serde = { version = "1.0.207", features = ["derive"] }
serde_json = "1.0.124" serde_json = "1.0.124"
serde_cbor = "0.11.2"
flate2 = "1.0.28" flate2 = "1.0.28"
lz4 = "1.24.0" lz4 = "1.24.0"
base64 = "0.22"
semver = { version = "1.0.20", features = ["serde"] } semver = { version = "1.0.20", features = ["serde"] }
diff-struct = "0.5.3" diff-struct = "0.5.3"
chrono = "0.4.41" chrono = "0.4.41"
tempfile = "3.20.0" tempfile = "3.20.0"
walkdir = "2.4.0" walkdir = "2.4.0"
redb = { version = "3" } rusqlite = { version = "0.31", default-features = false }
bincode = { version = "2", features = ["serde"] }
rust-ini = "0.21" rust-ini = "0.21"
reqwest = { version = "0.12", features = ["blocking", "json", "gzip", "deflate"] } reqwest = { version = "0.12", features = ["blocking", "json", "gzip", "deflate"] }
resolvo = "0.10" resolvo = "0.10"
[features] [features]
default = ["redb-index"] default = ["bundled-sqlite"]
redb-index = [] # Enable redb-based index for obsoleted packages bundled-sqlite = ["rusqlite/bundled"]

1165
libips/src/image/catalog.rs
View file

File diff suppressed because it is too large Load diff

338
libips/src/image/installed.rs
View file

@ -1,7 +1,8 @@
use crate::actions::Manifest; use crate::actions::Manifest;
use crate::fmri::Fmri; use crate::fmri::Fmri;
use crate::repository::sqlite_catalog::INSTALLED_SCHEMA;
use miette::Diagnostic; use miette::Diagnostic;
use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition}; use rusqlite::{Connection, OpenFlags};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::fs; use std::fs;
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
@ -9,11 +10,6 @@ use std::str::FromStr;
use thiserror::Error; use thiserror::Error;
use tracing::info; use tracing::info;
/// Table definition for the installed packages database
/// Key: full FMRI including publisher (pkg://publisher/stem@version)
/// Value: serialized Manifest
pub const INSTALLED_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("installed");
/// Errors that can occur when working with the installed packages database /// Errors that can occur when working with the installed packages database
#[derive(Error, Debug, Diagnostic)] #[derive(Error, Debug, Diagnostic)]
pub enum InstalledError { pub enum InstalledError {
@ -38,6 +34,12 @@ pub enum InstalledError {
PackageNotFound(String), PackageNotFound(String),
} }
impl From<rusqlite::Error> for InstalledError {
fn from(e: rusqlite::Error) -> Self {
InstalledError::Database(format!("SQLite error: {}", e))
}
}
/// Result type for installed packages operations /// Result type for installed packages operations
pub type Result<T> = std::result::Result<T, InstalledError>; pub type Result<T> = std::result::Result<T, InstalledError>;
@ -58,17 +60,6 @@ pub struct InstalledPackages {
} }
impl InstalledPackages { impl InstalledPackages {
// Note on borrowing and redb:
// When using redb, there's a potential borrowing issue when working with transactions and tables.
// The issue occurs because:
// 1. Tables borrow from the transaction they were opened from
// 2. When committing a transaction with tx.commit(), the transaction is moved
// 3. If a table is still borrowing from the transaction when commit() is called, Rust's borrow checker will prevent the move
//
// To fix this issue, we use block scopes {} around table operations to ensure that the table
// objects are dropped (and their borrows released) before committing the transaction.
// This pattern is used in all methods that commit transactions after table operations.
/// Create a new installed packages database /// Create a new installed packages database
pub fn new<P: AsRef<Path>>(db_path: P) -> Self { pub fn new<P: AsRef<Path>>(db_path: P) -> Self {
InstalledPackages { InstalledPackages {
@ -78,50 +69,26 @@ impl InstalledPackages {
/// Dump the contents of the installed table to stdout for debugging /// Dump the contents of the installed table to stdout for debugging
pub fn dump_installed_table(&self) -> Result<()> { pub fn dump_installed_table(&self) -> Result<()> {
// Open the database let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
// Begin a read transaction let mut stmt = conn.prepare("SELECT fmri, manifest FROM installed")?;
let tx = db let mut rows = stmt.query([])?;
.begin_read()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
// Open the installed table
match tx.open_table(INSTALLED_TABLE) {
Ok(table) => {
let mut count = 0; let mut count = 0;
for entry_result in table.iter().map_err(|e| { while let Some(row) = rows.next()? {
InstalledError::Database(format!("Failed to iterate installed table: {}", e)) let fmri_str: String = row.get(0)?;
})? { let manifest_bytes: Vec<u8> = row.get(1)?;
let (key, value) = entry_result.map_err(|e| {
InstalledError::Database(format!(
"Failed to get entry from installed table: {}",
e
))
})?;
let key_str = key.value();
// Try to deserialize the manifest match serde_json::from_slice::<Manifest>(&manifest_bytes) {
match serde_json::from_slice::<Manifest>(value.value()) {
Ok(manifest) => { Ok(manifest) => {
// Extract the publisher from the FMRI attribute println!("FMRI: {}", fmri_str);
let publisher = manifest
.attributes
.iter()
.find(|attr| attr.key == "pkg.fmri")
.and_then(|attr| attr.values.first().cloned())
.unwrap_or_else(|| "unknown".to_string());
println!("Key: {}", key_str);
println!(" FMRI: {}", publisher);
println!(" Attributes: {}", manifest.attributes.len()); println!(" Attributes: {}", manifest.attributes.len());
println!(" Files: {}", manifest.files.len()); println!(" Files: {}", manifest.files.len());
println!(" Directories: {}", manifest.directories.len()); println!(" Directories: {}", manifest.directories.len());
println!(" Dependencies: {}", manifest.dependencies.len()); println!(" Dependencies: {}", manifest.dependencies.len());
} }
Err(e) => { Err(e) => {
println!("Key: {}", key_str); println!("FMRI: {}", fmri_str);
println!(" Error deserializing manifest: {}", e); println!(" Error deserializing manifest: {}", e);
} }
} }
@ -130,46 +97,13 @@ impl InstalledPackages {
println!("Total entries in installed table: {}", count); println!("Total entries in installed table: {}", count);
Ok(()) Ok(())
} }
Err(e) => {
println!("Error opening installed table: {}", e);
Err(InstalledError::Database(format!(
"Failed to open installed table: {}",
e
)))
}
}
}
/// Get database statistics /// Get database statistics
pub fn get_db_stats(&self) -> Result<()> { pub fn get_db_stats(&self) -> Result<()> {
// Open the database let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
// Begin a read transaction let installed_count: i64 = conn.query_row("SELECT COUNT(*) FROM installed", [], |row| row.get(0))?;
let tx = db
.begin_read()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
// Get table statistics
let mut installed_count = 0;
// Count installed entries
if let Ok(table) = tx.open_table(INSTALLED_TABLE) {
for result in table.iter().map_err(|e| {
InstalledError::Database(format!("Failed to iterate installed table: {}", e))
})? {
let _ = result.map_err(|e| {
InstalledError::Database(format!(
"Failed to get entry from installed table: {}",
e
))
})?;
installed_count += 1;
}
}
// Print statistics
println!("Database path: {}", self.db_path.display()); println!("Database path: {}", self.db_path.display());
println!("Table statistics:"); println!("Table statistics:");
println!(" Installed table: {} entries", installed_count); println!(" Installed table: {} entries", installed_count);
@ -180,72 +114,33 @@ impl InstalledPackages {
/// Initialize the installed packages database /// Initialize the installed packages database
pub fn init_db(&self) -> Result<()> { pub fn init_db(&self) -> Result<()> {
// Create a parent directory if it doesn't exist // Create parent directory if it doesn't exist
if let Some(parent) = self.db_path.parent() { if let Some(parent) = self.db_path.parent() {
fs::create_dir_all(parent)?; fs::create_dir_all(parent)?;
} }
// Open or create the database // Create or open the database
let db = Database::create(&self.db_path) let conn = Connection::open(&self.db_path)?;
.map_err(|e| InstalledError::Database(format!("Failed to create database: {}", e)))?;
// Create tables // Execute schema
let tx = db conn.execute_batch(INSTALLED_SCHEMA)?;
.begin_write()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
tx.open_table(INSTALLED_TABLE).map_err(|e| {
InstalledError::Database(format!("Failed to create installed table: {}", e))
})?;
tx.commit().map_err(|e| {
InstalledError::Database(format!("Failed to commit transaction: {}", e))
})?;
Ok(()) Ok(())
} }
/// Add a package to the installed packages database /// Add a package to the installed packages database
pub fn add_package(&self, fmri: &Fmri, manifest: &Manifest) -> Result<()> { pub fn add_package(&self, fmri: &Fmri, manifest: &Manifest) -> Result<()> {
// Open the database let mut conn = Connection::open(&self.db_path)?;
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
// Begin a writing transaction
let tx = db
.begin_write()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
// Create the key (full FMRI including publisher)
let key = fmri.to_string(); let key = fmri.to_string();
// Serialize the manifest
let manifest_bytes = serde_json::to_vec(manifest)?; let manifest_bytes = serde_json::to_vec(manifest)?;
// Use a block scope to ensure the table is dropped before committing the transaction let tx = conn.transaction()?;
{ tx.execute(
// Open the installed table "INSERT OR REPLACE INTO installed (fmri, manifest) VALUES (?1, ?2)",
let mut installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| { rusqlite::params![key, manifest_bytes],
InstalledError::Database(format!("Failed to open installed table: {}", e)) )?;
})?; tx.commit()?;
// Insert the package into the installed table
installed_table
.insert(key.as_str(), manifest_bytes.as_slice())
.map_err(|e| {
InstalledError::Database(format!(
"Failed to insert into installed table: {}",
e
))
})?;
// The table is dropped at the end of this block, releasing its borrow of tx
}
// Commit the transaction
tx.commit().map_err(|e| {
InstalledError::Database(format!("Failed to commit transaction: {}", e))
})?;
info!("Added package to installed database: {}", key); info!("Added package to installed database: {}", key);
Ok(()) Ok(())
@ -253,42 +148,24 @@ impl InstalledPackages {
/// Remove a package from the installed packages database /// Remove a package from the installed packages database
pub fn remove_package(&self, fmri: &Fmri) -> Result<()> { pub fn remove_package(&self, fmri: &Fmri) -> Result<()> {
// Open the database let mut conn = Connection::open(&self.db_path)?;
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
// Begin a writing transaction
let tx = db
.begin_write()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
// Create the key (full FMRI including publisher)
let key = fmri.to_string(); let key = fmri.to_string();
// Use a block scope to ensure the table is dropped before committing the transaction
{
// Open the installed table
let mut installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| {
InstalledError::Database(format!("Failed to open installed table: {}", e))
})?;
// Check if the package exists // Check if the package exists
if let Ok(None) = installed_table.get(key.as_str()) { let exists: bool = conn.query_row(
"SELECT EXISTS(SELECT 1 FROM installed WHERE fmri = ?1)",
rusqlite::params![key],
|row| row.get(0),
)?;
if !exists {
return Err(InstalledError::PackageNotFound(key)); return Err(InstalledError::PackageNotFound(key));
} }
// Remove the package from the installed table let tx = conn.transaction()?;
installed_table.remove(key.as_str()).map_err(|e| { tx.execute("DELETE FROM installed WHERE fmri = ?1", rusqlite::params![key])?;
InstalledError::Database(format!("Failed to remove from installed table: {}", e)) tx.commit()?;
})?;
// The table is dropped at the end of this block, releasing its borrow of tx
}
// Commit the transaction
tx.commit().map_err(|e| {
InstalledError::Database(format!("Failed to commit transaction: {}", e))
})?;
info!("Removed package from installed database: {}", key); info!("Removed package from installed database: {}", key);
Ok(()) Ok(())
@ -296,127 +173,60 @@ impl InstalledPackages {
/// Query the installed packages database for packages matching a pattern /// Query the installed packages database for packages matching a pattern
pub fn query_packages(&self, pattern: Option<&str>) -> Result<Vec<InstalledPackageInfo>> { pub fn query_packages(&self, pattern: Option<&str>) -> Result<Vec<InstalledPackageInfo>> {
// Open the database let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
// Begin a read transaction let query = if let Some(pattern) = pattern {
let tx = db format!("SELECT fmri FROM installed WHERE fmri LIKE '%{}%'", pattern.replace('\'', "''"))
.begin_read() } else {
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?; "SELECT fmri FROM installed".to_string()
};
// Use a block scope to ensure the table is dropped when no longer needed let mut stmt = conn.prepare(&query)?;
let results = { let mut rows = stmt.query([])?;
// Open the installed table
let installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| {
InstalledError::Database(format!("Failed to open installed table: {}", e))
})?;
let mut results = Vec::new(); let mut results = Vec::new();
while let Some(row) = rows.next()? {
// Process the installed table let fmri_str: String = row.get(0)?;
// Iterate through all entries in the table let fmri = Fmri::from_str(&fmri_str)?;
for entry_result in installed_table.iter().map_err(|e| { let publisher = fmri.publisher.clone().unwrap_or_else(|| "unknown".to_string());
InstalledError::Database(format!("Failed to iterate installed table: {}", e))
})? {
let (key, _) = entry_result.map_err(|e| {
InstalledError::Database(format!(
"Failed to get entry from installed table: {}",
e
))
})?;
let key_str = key.value();
// Skip if the key doesn't match the pattern
if let Some(pattern) = pattern {
if !key_str.contains(pattern) {
continue;
}
}
// Parse the key to get the FMRI
let fmri = Fmri::from_str(key_str)?;
// Get the publisher (handling the Option<String>)
let publisher = fmri
.publisher
.clone()
.unwrap_or_else(|| "unknown".to_string());
// Add to results
results.push(InstalledPackageInfo { fmri, publisher }); results.push(InstalledPackageInfo { fmri, publisher });
} }
results
// The table is dropped at the end of this block
};
Ok(results) Ok(results)
} }
/// Get a manifest from the installed packages database /// Get a manifest from the installed packages database
pub fn get_manifest(&self, fmri: &Fmri) -> Result<Option<Manifest>> { pub fn get_manifest(&self, fmri: &Fmri) -> Result<Option<Manifest>> {
// Open the database let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
// Begin a read transaction
let tx = db
.begin_read()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
// Create the key (full FMRI including publisher)
let key = fmri.to_string(); let key = fmri.to_string();
let result = conn.query_row(
"SELECT manifest FROM installed WHERE fmri = ?1",
rusqlite::params![key],
|row| {
let bytes: Vec<u8> = row.get(0)?;
Ok(bytes)
},
);
// Use a block scope to ensure the table is dropped when no longer needed match result {
let manifest_option = { Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes)?)),
// Open the installed table Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
let installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| { Err(e) => Err(e.into()),
InstalledError::Database(format!("Failed to open installed table: {}", e))
})?;
// Try to get the manifest from the installed table
if let Ok(Some(bytes)) = installed_table.get(key.as_str()) {
Some(serde_json::from_slice(bytes.value())?)
} else {
None
} }
// The table is dropped at the end of this block
};
Ok(manifest_option)
} }
/// Check if a package is installed /// Check if a package is installed
pub fn is_installed(&self, fmri: &Fmri) -> Result<bool> { pub fn is_installed(&self, fmri: &Fmri) -> Result<bool> {
// Open the database let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
// Begin a read transaction
let tx = db
.begin_read()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
// Create the key (full FMRI including publisher)
let key = fmri.to_string(); let key = fmri.to_string();
let exists: bool = conn.query_row(
"SELECT EXISTS(SELECT 1 FROM installed WHERE fmri = ?1)",
rusqlite::params![key],
|row| row.get(0),
)?;
// Use a block scope to ensure the table is dropped when no longer needed Ok(exists)
let is_installed = {
// Open the installed table
let installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| {
InstalledError::Database(format!("Failed to open installed table: {}", e))
})?;
// Check if the package exists
if let Ok(Some(_)) = installed_table.get(key.as_str()) {
true
} else {
false
}
// The table is dropped at the end of this block
};
Ok(is_installed)
} }
} }

18
libips/src/image/installed_tests.rs
View file

@ -1,7 +1,7 @@
use super::*; use super::*;
use crate::actions::{Attr, Manifest}; use crate::actions::{Attr, Manifest};
use crate::fmri::Fmri; use crate::fmri::Fmri;
use redb::{Database, ReadableTable}; use rusqlite::Connection;
use std::str::FromStr; use std::str::FromStr;
use tempfile::tempdir; use tempfile::tempdir;
@ -82,7 +82,7 @@ fn test_installed_packages() {
fn test_installed_packages_key_format() { fn test_installed_packages_key_format() {
// Create a temporary directory for the test // Create a temporary directory for the test
let temp_dir = tempdir().unwrap(); let temp_dir = tempdir().unwrap();
let db_path = temp_dir.path().join("installed.redb"); let db_path = temp_dir.path().join("installed.db");
// Create the installed packages database // Create the installed packages database
let installed = InstalledPackages::new(&db_path); let installed = InstalledPackages::new(&db_path);
@ -104,15 +104,15 @@ fn test_installed_packages_key_format() {
installed.add_package(&fmri, &manifest).unwrap(); installed.add_package(&fmri, &manifest).unwrap();
// Open the database directly to check the key format // Open the database directly to check the key format
let db = Database::open(&db_path).unwrap(); let conn = Connection::open(&db_path).unwrap();
let tx = db.begin_read().unwrap(); let mut stmt = conn.prepare("SELECT fmri FROM installed").unwrap();
let table = tx.open_table(installed::INSTALLED_TABLE).unwrap(); let mut rows = stmt.query([]).unwrap();
// Iterate through the keys // Collect the keys
let mut keys = Vec::new(); let mut keys = Vec::new();
for entry in table.iter().unwrap() { while let Some(row) = rows.next().unwrap() {
let (key, _) = entry.unwrap(); let fmri_str: String = row.get(0).unwrap();
keys.push(key.value().to_string()); keys.push(fmri_str);
} }
// Verify that there is one key and it has the correct format // Verify that there is one key and it has the correct format

166
libips/src/image/mod.rs
View file

@ -4,7 +4,7 @@ mod tests;
use miette::Diagnostic; use miette::Diagnostic;
use properties::*; use properties::*;
use redb::{Database, ReadableDatabase, ReadableTable}; use rusqlite::{Connection, OpenFlags};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashMap; use std::collections::HashMap;
use std::fs::{self, File}; use std::fs::{self, File};
@ -15,7 +15,7 @@ use crate::repository::{FileBackend, ReadableRepository, RepositoryError, RestBa
// Export the catalog module // Export the catalog module
pub mod catalog; pub mod catalog;
use catalog::{INCORPORATE_TABLE, ImageCatalog, PackageInfo}; use catalog::{ImageCatalog, PackageInfo};
// Export the installed packages module // Export the installed packages module
pub mod installed; pub mod installed;
@ -79,6 +79,12 @@ pub enum ImageError {
NoPublishers, NoPublishers,
} }
impl From<rusqlite::Error> for ImageError {
fn from(e: rusqlite::Error) -> Self {
ImageError::Database(format!("SQLite error: {}", e))
}
}
pub type Result<T> = std::result::Result<T, ImageError>; pub type Result<T> = std::result::Result<T, ImageError>;
/// Type of image, either Full (base path of "/") or Partial (attached to a full image) /// Type of image, either Full (base path of "/") or Partial (attached to a full image)
@ -281,7 +287,7 @@ impl Image {
/// Returns the path to the installed packages database /// Returns the path to the installed packages database
pub fn installed_db_path(&self) -> PathBuf { pub fn installed_db_path(&self) -> PathBuf {
self.metadata_dir().join("installed.redb") self.metadata_dir().join("installed.db")
} }
/// Returns the path to the manifest directory /// Returns the path to the manifest directory
@ -294,14 +300,31 @@ impl Image {
self.metadata_dir().join("catalog") self.metadata_dir().join("catalog")
} }
/// Returns the path to the catalog database /// Returns the path to the active catalog database (packages and dependencies)
pub fn catalog_db_path(&self) -> PathBuf { pub fn active_db_path(&self) -> PathBuf {
self.metadata_dir().join("catalog.redb") self.metadata_dir().join("active.db")
} }
/// Returns the path to the obsoleted packages database (separate DB) /// Returns the path to the obsoleted packages database
pub fn obsolete_db_path(&self) -> PathBuf {
self.metadata_dir().join("obsolete.db")
}
/// Returns the path to the full-text search database
pub fn fts_db_path(&self) -> PathBuf {
self.metadata_dir().join("fts.db")
}
/// Deprecated: Use active_db_path() instead
#[deprecated(note = "Use active_db_path() instead")]
pub fn catalog_db_path(&self) -> PathBuf {
self.active_db_path()
}
/// Deprecated: Use obsolete_db_path() instead
#[deprecated(note = "Use obsolete_db_path() instead")]
pub fn obsoleted_db_path(&self) -> PathBuf { pub fn obsoleted_db_path(&self) -> PathBuf {
self.metadata_dir().join("obsoleted.redb") self.obsolete_db_path()
} }
/// Creates the metadata directory if it doesn't exist /// Creates the metadata directory if it doesn't exist
@ -528,14 +551,18 @@ impl Image {
/// Initialize the catalog database /// Initialize the catalog database
pub fn init_catalog_db(&self) -> Result<()> { pub fn init_catalog_db(&self) -> Result<()> {
let catalog = ImageCatalog::new( use crate::repository::sqlite_catalog::ACTIVE_SCHEMA;
self.catalog_dir(),
self.catalog_db_path(), let path = self.active_db_path();
self.obsoleted_db_path(), if let Some(parent) = path.parent() {
); fs::create_dir_all(parent)?;
catalog.init_db().map_err(|e| { }
ImageError::Database(format!("Failed to initialize catalog database: {}", e))
}) let conn = Connection::open(&path)?;
conn.execute_batch(ACTIVE_SCHEMA)
.map_err(|e| ImageError::Database(format!("Failed to initialize catalog database: {}", e)))?;
Ok(())
} }
/// Download catalogs from all configured publishers and build the merged catalog /// Download catalogs from all configured publishers and build the merged catalog
@ -657,65 +684,102 @@ impl Image {
/// Look up an incorporation lock for a given stem. /// Look up an incorporation lock for a given stem.
/// Returns Some(release) if a lock exists, otherwise None. /// Returns Some(release) if a lock exists, otherwise None.
pub fn get_incorporated_release(&self, stem: &str) -> Result<Option<String>> { pub fn get_incorporated_release(&self, stem: &str) -> Result<Option<String>> {
let db = Database::open(self.catalog_db_path()) let conn = Connection::open_with_flags(
&self.active_db_path(),
OpenFlags::SQLITE_OPEN_READ_ONLY,
)
.map_err(|e| ImageError::Database(format!("Failed to open catalog database: {}", e)))?; .map_err(|e| ImageError::Database(format!("Failed to open catalog database: {}", e)))?;
let tx = db.begin_read().map_err(|e| {
ImageError::Database(format!("Failed to begin read transaction: {}", e)) let result = conn.query_row(
})?; "SELECT release FROM incorporate_locks WHERE stem = ?1",
match tx.open_table(INCORPORATE_TABLE) { rusqlite::params![stem],
Ok(table) => match table.get(stem) { |row| row.get(0),
Ok(Some(val)) => Ok(Some(String::from_utf8_lossy(val.value()).to_string())), );
Ok(None) => Ok(None),
match result {
Ok(release) => Ok(Some(release)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(ImageError::Database(format!( Err(e) => Err(ImageError::Database(format!(
"Failed to read incorporate lock: {}", "Failed to read incorporate lock: {}",
e e
))), ))),
},
Err(_) => Ok(None),
} }
} }
/// Add an incorporation lock for a stem to a specific release. /// Add an incorporation lock for a stem to a specific release.
/// Fails if a lock already exists for the stem. /// Uses INSERT OR REPLACE, so will update if a lock already exists.
pub fn add_incorporation_lock(&self, stem: &str, release: &str) -> Result<()> { pub fn add_incorporation_lock(&self, stem: &str, release: &str) -> Result<()> {
let db = Database::open(self.catalog_db_path()) let mut conn = Connection::open(&self.active_db_path())
.map_err(|e| ImageError::Database(format!("Failed to open catalog database: {}", e)))?; .map_err(|e| ImageError::Database(format!("Failed to open catalog database: {}", e)))?;
let tx = db.begin_write().map_err(|e| {
let tx = conn.transaction().map_err(|e| {
ImageError::Database(format!("Failed to begin write transaction: {}", e)) ImageError::Database(format!("Failed to begin write transaction: {}", e))
})?; })?;
{
let mut table = tx.open_table(INCORPORATE_TABLE).map_err(|e| { tx.execute(
ImageError::Database(format!("Failed to open incorporate table: {}", e)) "INSERT OR REPLACE INTO incorporate_locks (stem, release) VALUES (?1, ?2)",
})?; rusqlite::params![stem, release],
if let Ok(Some(_)) = table.get(stem) { )
return Err(ImageError::Database(format!( .map_err(|e| ImageError::Database(format!("Failed to insert incorporate lock: {}", e)))?;
"Incorporation lock already exists for stem {}",
stem
)));
}
table.insert(stem, release.as_bytes()).map_err(|e| {
ImageError::Database(format!("Failed to insert incorporate lock: {}", e))
})?;
}
tx.commit().map_err(|e| { tx.commit().map_err(|e| {
ImageError::Database(format!("Failed to commit incorporate lock: {}", e)) ImageError::Database(format!("Failed to commit incorporate lock: {}", e))
})?; })?;
Ok(()) Ok(())
} }
/// Get a manifest from the catalog /// Get a manifest from the catalog.
/// First checks the local manifest cache on disk, then falls back to repository fetch.
/// Note: active.db does NOT store manifest blobs - manifests are served from the repository.
pub fn get_manifest_from_catalog( pub fn get_manifest_from_catalog(
&self, &self,
fmri: &crate::fmri::Fmri, fmri: &crate::fmri::Fmri,
) -> Result<Option<crate::actions::Manifest>> { ) -> Result<Option<crate::actions::Manifest>> {
let catalog = ImageCatalog::new( // Helper to URL-encode filename components
self.catalog_dir(), fn url_encode(s: &str) -> String {
self.catalog_db_path(), s.chars()
self.obsoleted_db_path(), .map(|c| match c {
); 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(),
catalog.get_manifest(fmri).map_err(|e| { ' ' => "+".to_string(),
ImageError::Database(format!("Failed to get manifest from catalog: {}", e)) _ => {
let mut buf = [0u8; 4];
let bytes = c.encode_utf8(&mut buf).as_bytes();
bytes.iter().map(|b| format!("%{:02X}", b)).collect()
}
}) })
.collect()
}
// Check local manifest cache on disk
let publisher = fmri.publisher.as_deref().unwrap_or("");
let manifest_dir = self.manifest_dir().join(publisher);
let stem_encoded = url_encode(fmri.stem());
let version_encoded = url_encode(&fmri.version());
let manifest_path = manifest_dir.join(format!("{}@{}.p5m", stem_encoded, version_encoded));
if manifest_path.exists() {
let content = fs::read_to_string(&manifest_path)?;
let manifest = crate::actions::Manifest::parse_string(content)
.map_err(|e| ImageError::Database(format!("Failed to parse manifest: {}", e)))?;
return Ok(Some(manifest));
}
// Check catalog shards for a minimal manifest
let catalog = crate::image::catalog::ImageCatalog::new(
self.catalog_dir(),
self.active_db_path(),
self.obsolete_db_path(),
);
if let Ok(Some(manifest)) = catalog.get_manifest(fmri) {
return Ok(Some(manifest));
}
// Fall back to repository fetch
match self.get_manifest_from_repository(fmri) {
Ok(manifest) => Ok(Some(manifest)),
Err(_) => Ok(None),
}
} }
/// Fetch a full manifest for the given FMRI directly from its repository origin. /// Fetch a full manifest for the given FMRI directly from its repository origin.

43
libips/src/repository/file_backend.rs
View file

@ -1899,11 +1899,25 @@ impl WritableRepository for FileBackend {
if !no_catalog { if !no_catalog {
info!("Rebuilding catalog..."); info!("Rebuilding catalog...");
self.rebuild_catalog(&pub_name, true)?; self.rebuild_catalog(&pub_name, true)?;
// Build SQLite catalog shards (active.db, obsolete.db, fts.db)
info!("Building catalog shards...");
let catalog_dir = Self::construct_catalog_path(&self.path, &pub_name);
let shard_dir = self.shard_dir(&pub_name);
crate::repository::sqlite_catalog::build_shards(
&catalog_dir,
&pub_name,
&shard_dir,
)
.map_err(|e| {
RepositoryError::Other(format!("Failed to build catalog shards: {}", e.message))
})?;
} }
if !no_index { if !no_index {
info!("Rebuilding search index..."); // FTS index is now built as part of catalog shards (fts.db)
self.build_search_index(&pub_name)?; // No separate index building needed
info!("Search index built as part of catalog shards (fts.db)");
} }
} }
@ -1929,11 +1943,25 @@ impl WritableRepository for FileBackend {
if !no_catalog { if !no_catalog {
info!("Refreshing catalog..."); info!("Refreshing catalog...");
self.rebuild_catalog(&pub_name, true)?; self.rebuild_catalog(&pub_name, true)?;
// Build SQLite catalog shards (active.db, obsolete.db, fts.db)
info!("Building catalog shards...");
let catalog_dir = Self::construct_catalog_path(&self.path, &pub_name);
let shard_dir = self.shard_dir(&pub_name);
crate::repository::sqlite_catalog::build_shards(
&catalog_dir,
&pub_name,
&shard_dir,
)
.map_err(|e| {
RepositoryError::Other(format!("Failed to build catalog shards: {}", e.message))
})?;
} }
if !no_index { if !no_index {
info!("Refreshing search index..."); // FTS index is now built as part of catalog shards (fts.db)
self.build_search_index(&pub_name)?; // No separate index building needed
info!("Search index built as part of catalog shards (fts.db)");
} }
} }
@ -2166,6 +2194,13 @@ impl FileBackend {
base_path.join("publisher").join(publisher).join("catalog") base_path.join("publisher").join(publisher).join("catalog")
} }
/// Helper method to construct a shard directory path for catalog v2 shards
///
/// Format: base_path/publisher/publisher_name/catalog2
pub fn shard_dir(&self, publisher: &str) -> PathBuf {
self.path.join("publisher").join(publisher).join("catalog2")
}
/// Helper method to construct a manifest path consistently /// Helper method to construct a manifest path consistently
/// ///
/// Format: base_path/publisher/publisher_name/pkg/stem/encoded_version /// Format: base_path/publisher/publisher_name/pkg/stem/encoded_version

49
libips/src/repository/mod.rs
View file

@ -168,60 +168,21 @@ impl From<StripPrefixError> for RepositoryError {
} }
} }
// Implement From for redb error types // Implement From for rusqlite error types
impl From<redb::Error> for RepositoryError { impl From<rusqlite::Error> for RepositoryError {
fn from(err: redb::Error) -> Self { fn from(err: rusqlite::Error) -> Self {
RepositoryError::Other(format!("Database error: {}", err)) RepositoryError::Other(format!("Database error: {}", err))
} }
} }
impl From<redb::DatabaseError> for RepositoryError {
fn from(err: redb::DatabaseError) -> Self {
RepositoryError::Other(format!("Database error: {}", err))
}
}
impl From<redb::TransactionError> for RepositoryError {
fn from(err: redb::TransactionError) -> Self {
RepositoryError::Other(format!("Transaction error: {}", err))
}
}
impl From<redb::TableError> for RepositoryError {
fn from(err: redb::TableError) -> Self {
RepositoryError::Other(format!("Table error: {}", err))
}
}
impl From<redb::StorageError> for RepositoryError {
fn from(err: redb::StorageError) -> Self {
RepositoryError::Other(format!("Storage error: {}", err))
}
}
impl From<redb::CommitError> for RepositoryError {
fn from(err: redb::CommitError) -> Self {
RepositoryError::Other(format!("Commit error: {}", err))
}
}
impl From<bincode::error::DecodeError> for RepositoryError {
fn from(err: bincode::error::DecodeError) -> Self {
RepositoryError::Other(format!("Serialization error: {}", err))
}
}
impl From<bincode::error::EncodeError> for RepositoryError {
fn from(err: bincode::error::EncodeError) -> Self {
RepositoryError::Other(format!("Serialization error: {}", err))
}
}
pub mod catalog; pub mod catalog;
mod catalog_writer; mod catalog_writer;
pub(crate) mod file_backend; pub(crate) mod file_backend;
mod obsoleted; mod obsoleted;
pub mod progress; pub mod progress;
mod rest_backend; mod rest_backend;
pub mod shard_sync;
pub mod sqlite_catalog;
#[cfg(test)] #[cfg(test)]
mod tests; mod tests;

544
libips/src/repository/obsoleted.rs
View file

@ -1,11 +1,11 @@
use crate::fmri::Fmri; use crate::fmri::Fmri;
use crate::repository::sqlite_catalog::OBSOLETED_INDEX_SCHEMA;
use crate::repository::{RepositoryError, Result}; use crate::repository::{RepositoryError, Result};
use chrono::{DateTime, Duration as ChronoDuration, Utc}; use chrono::{DateTime, Duration as ChronoDuration, Utc};
use miette::Diagnostic; use miette::Diagnostic;
use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition};
use regex::Regex; use regex::Regex;
use rusqlite::{Connection, OpenFlags};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_cbor;
use serde_json; use serde_json;
use sha2::Digest; use sha2::Digest;
use std::fs; use std::fs;
@ -176,54 +176,12 @@ impl From<crate::fmri::FmriError> for ObsoletedPackageError {
} }
} }
impl From<redb::Error> for ObsoletedPackageError { impl From<rusqlite::Error> for ObsoletedPackageError {
fn from(err: redb::Error) -> Self { fn from(err: rusqlite::Error) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string()) ObsoletedPackageError::DatabaseError(err.to_string())
} }
} }
impl From<redb::DatabaseError> for ObsoletedPackageError {
fn from(err: redb::DatabaseError) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string())
}
}
impl From<redb::TransactionError> for ObsoletedPackageError {
fn from(err: redb::TransactionError) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string())
}
}
impl From<redb::TableError> for ObsoletedPackageError {
fn from(err: redb::TableError) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string())
}
}
impl From<redb::StorageError> for ObsoletedPackageError {
fn from(err: redb::StorageError) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string())
}
}
impl From<redb::CommitError> for ObsoletedPackageError {
fn from(err: redb::CommitError) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string())
}
}
impl From<bincode::error::EncodeError> for ObsoletedPackageError {
fn from(err: bincode::error::EncodeError) -> Self {
ObsoletedPackageError::SerializationError(err.to_string())
}
}
impl From<bincode::error::DecodeError> for ObsoletedPackageError {
fn from(err: bincode::error::DecodeError) -> Self {
ObsoletedPackageError::SerializationError(err.to_string())
}
}
// Implement From<ObsoletedPackageError> for RepositoryError to allow conversion // Implement From<ObsoletedPackageError> for RepositoryError to allow conversion
// This makes it easier to use ObsoletedPackageError with the existing Result type // This makes it easier to use ObsoletedPackageError with the existing Result type
impl From<ObsoletedPackageError> for RepositoryError { impl From<ObsoletedPackageError> for RepositoryError {
@ -325,19 +283,11 @@ impl ObsoletedPackageKey {
} }
} }
// Table definitions for the redb database /// Index of obsoleted packages using SQLite for faster lookups and content-addressable storage
// Table for mapping FMRI directly to metadata
static FMRI_TO_METADATA_TABLE: TableDefinition<&[u8], &[u8]> =
TableDefinition::new("fmri_to_metadata");
// Table for mapping content hash to manifest (for non-NULL_HASH entries)
static HASH_TO_MANIFEST_TABLE: TableDefinition<&str, &str> =
TableDefinition::new("hash_to_manifest");
/// Index of obsoleted packages using redb for faster lookups and content-addressable storage
#[derive(Debug)] #[derive(Debug)]
struct RedbObsoletedPackageIndex { struct SqliteObsoletedPackageIndex {
/// The redb database /// Path to the SQLite database file
db: Database, db_path: PathBuf,
/// Last time the index was accessed /// Last time the index was accessed
last_accessed: Instant, last_accessed: Instant,
/// Whether the index is dirty and needs to be rebuilt /// Whether the index is dirty and needs to be rebuilt
@ -346,26 +296,18 @@ struct RedbObsoletedPackageIndex {
max_age: Duration, max_age: Duration,
} }
impl RedbObsoletedPackageIndex { impl SqliteObsoletedPackageIndex {
/// Create a new RedbObsoletedPackageIndex /// Create a new SqliteObsoletedPackageIndex
fn new<P: AsRef<Path>>(base_path: P) -> Result<Self> { fn new<P: AsRef<Path>>(base_path: P) -> Result<Self> {
let db_path = base_path.as_ref().join("index.redb"); let db_path = base_path.as_ref().join("index.db");
debug!("Creating redb database at {}", db_path.display()); debug!("Creating SQLite database at {}", db_path.display());
// Create the database // Create the database and tables
let db = Database::create(&db_path)?; let conn = Connection::open(&db_path)?;
conn.execute_batch(OBSOLETED_INDEX_SCHEMA)?;
// Create the tables if they don't exist
let write_txn = db.begin_write()?;
{
// Create the new table for direct FMRI to metadata mapping
write_txn.open_table(FMRI_TO_METADATA_TABLE)?;
write_txn.open_table(HASH_TO_MANIFEST_TABLE)?;
}
write_txn.commit()?;
Ok(Self { Ok(Self {
db, db_path,
last_accessed: Instant::now(), last_accessed: Instant::now(),
dirty: false, dirty: false,
max_age: Duration::from_secs(300), // 5 minutes max_age: Duration::from_secs(300), // 5 minutes
@ -377,12 +319,12 @@ impl RedbObsoletedPackageIndex {
self.dirty || self.last_accessed.elapsed() > self.max_age self.dirty || self.last_accessed.elapsed() > self.max_age
} }
/// Create an empty temporary file-based RedbObsoletedPackageIndex /// Create an empty temporary file-based SqliteObsoletedPackageIndex
/// ///
/// This is used as a fallback when the database creation fails. /// This is used as a fallback when the database creation fails.
/// It creates a database in a temporary directory that can be used temporarily. /// It creates a database in a temporary directory that can be used temporarily.
fn empty() -> Self { fn empty() -> Self {
debug!("Creating empty temporary file-based redb database"); debug!("Creating empty temporary file-based SQLite database");
// Create a temporary directory // Create a temporary directory
let temp_dir = tempfile::tempdir().unwrap_or_else(|e| { let temp_dir = tempfile::tempdir().unwrap_or_else(|e| {
@ -391,50 +333,44 @@ impl RedbObsoletedPackageIndex {
}); });
// Create a database file in the temporary directory // Create a database file in the temporary directory
let db_path = temp_dir.path().join("empty.redb"); let db_path = temp_dir.path().join("empty.db");
// Create the database // Create the database and tables
let db = Database::create(&db_path).unwrap_or_else(|e| { let conn = Connection::open(&db_path).unwrap_or_else(|e| {
error!("Failed to create temporary database: {}", e); error!("Failed to create temporary database: {}", e);
panic!("Failed to create temporary database: {}", e); panic!("Failed to create temporary database: {}", e);
}); });
// Create the tables conn.execute_batch(OBSOLETED_INDEX_SCHEMA).unwrap();
let write_txn = db.begin_write().unwrap();
{
// Create the new table for direct FMRI to metadata mapping
let _ = write_txn.open_table(FMRI_TO_METADATA_TABLE).unwrap();
let _ = write_txn.open_table(HASH_TO_MANIFEST_TABLE).unwrap();
}
write_txn.commit().unwrap();
Self { Self {
db, db_path,
last_accessed: Instant::now(), last_accessed: Instant::now(),
dirty: false, dirty: false,
max_age: Duration::from_secs(300), // 5 minutes max_age: Duration::from_secs(300), // 5 minutes
} }
} }
/// Open an existing RedbObsoletedPackageIndex /// Open an existing SqliteObsoletedPackageIndex
fn open<P: AsRef<Path>>(base_path: P) -> Result<Self> { fn open<P: AsRef<Path>>(base_path: P) -> Result<Self> {
let db_path = base_path.as_ref().join("index.redb"); let db_path = base_path.as_ref().join("index.db");
debug!("Opening redb database at {}", db_path.display()); debug!("Opening SQLite database at {}", db_path.display());
// Open the database // Open the database (creating tables if they don't exist)
let db = Database::open(&db_path)?; let conn = Connection::open(&db_path)?;
conn.execute_batch(OBSOLETED_INDEX_SCHEMA)?;
Ok(Self { Ok(Self {
db, db_path,
last_accessed: Instant::now(), last_accessed: Instant::now(),
dirty: false, dirty: false,
max_age: Duration::from_secs(300), // 5 minutes max_age: Duration::from_secs(300), // 5 minutes
}) })
} }
/// Create or open a RedbObsoletedPackageIndex /// Create or open a SqliteObsoletedPackageIndex
fn create_or_open<P: AsRef<Path>>(base_path: P) -> Result<Self> { fn create_or_open<P: AsRef<Path>>(base_path: P) -> Result<Self> {
let db_path = base_path.as_ref().join("index.redb"); let db_path = base_path.as_ref().join("index.db");
if db_path.exists() { if db_path.exists() {
Self::open(base_path) Self::open(base_path)
@ -464,69 +400,46 @@ impl RedbObsoletedPackageIndex {
metadata.content_hash.clone() metadata.content_hash.clone()
}; };
// Use the FMRI string directly as the key // Serialize obsoleted_by as JSON string (or NULL if None)
let key_bytes = metadata.fmri.as_bytes(); let obsoleted_by_json = metadata
.obsoleted_by
.as_ref()
.map(|obs| serde_json::to_string(obs))
.transpose()?;
let metadata_bytes = match serde_cbor::to_vec(metadata) { let mut conn = Connection::open(&self.db_path)?;
Ok(bytes) => bytes, let tx = conn.transaction()?;
Err(e) => {
error!("Failed to serialize metadata with CBOR: {}", e);
return Err(ObsoletedPackageError::SerializationError(format!(
"Failed to serialize metadata with CBOR: {}",
e
))
.into());
}
};
// Begin write transaction // Insert into obsoleted_packages table
let write_txn = match self.db.begin_write() { tx.execute(
Ok(txn) => txn, "INSERT OR REPLACE INTO obsoleted_packages (
Err(e) => { fmri, publisher, stem, version, status, obsolescence_date,
error!("Failed to begin write transaction: {}", e); deprecation_message, obsoleted_by, metadata_version, content_hash
return Err(e.into()); ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
} rusqlite::params![
}; &metadata.fmri,
&key.publisher,
{ &key.stem,
// Open the tables &key.version,
let mut fmri_to_metadata = match write_txn.open_table(FMRI_TO_METADATA_TABLE) { &metadata.status,
Ok(table) => table, &metadata.obsolescence_date,
Err(e) => { metadata.deprecation_message.as_deref(),
error!("Failed to open FMRI_TO_METADATA_TABLE: {}", e); obsoleted_by_json.as_deref(),
return Err(e.into()); metadata.metadata_version,
} &content_hash,
}; ],
)?;
let mut hash_to_manifest = match write_txn.open_table(HASH_TO_MANIFEST_TABLE) {
Ok(table) => table,
Err(e) => {
error!("Failed to open HASH_TO_MANIFEST_TABLE: {}", e);
return Err(e.into());
}
};
// Insert the metadata directly with FMRI as the key
// This is the new approach that eliminates the intermediate hash lookup
if let Err(e) = fmri_to_metadata.insert(key_bytes, metadata_bytes.as_slice()) {
error!("Failed to insert into FMRI_TO_METADATA_TABLE: {}", e);
return Err(e.into());
}
// Only store the manifest if it's not a NULL_HASH entry // Only store the manifest if it's not a NULL_HASH entry
// For NULL_HASH entries, a minimal manifest will be generated when requested // For NULL_HASH entries, a minimal manifest will be generated when requested
if content_hash != NULL_HASH { if content_hash != NULL_HASH {
if let Err(e) = hash_to_manifest.insert(content_hash.as_str(), manifest) { tx.execute(
error!("Failed to insert into HASH_TO_MANIFEST_TABLE: {}", e); "INSERT OR REPLACE INTO obsoleted_manifests (content_hash, manifest) VALUES (?1, ?2)",
return Err(e.into()); rusqlite::params![&content_hash, manifest],
} )?;
}
} }
if let Err(e) = write_txn.commit() { tx.commit()?;
error!("Failed to commit transaction: {}", e);
return Err(e.into());
}
debug!("Successfully added entry to index: {}", metadata.fmri); debug!("Successfully added entry to index: {}", metadata.fmri);
Ok(()) Ok(())
@ -536,32 +449,25 @@ impl RedbObsoletedPackageIndex {
fn remove_entry(&self, key: &ObsoletedPackageKey) -> Result<bool> { fn remove_entry(&self, key: &ObsoletedPackageKey) -> Result<bool> {
// Use the FMRI string directly as the key // Use the FMRI string directly as the key
let fmri = key.to_fmri_string(); let fmri = key.to_fmri_string();
let key_bytes = fmri.as_bytes();
// First, check if the key exists in the new table let conn = Connection::open(&self.db_path)?;
let exists_in_new_table = {
let read_txn = self.db.begin_read()?;
let fmri_to_metadata = read_txn.open_table(FMRI_TO_METADATA_TABLE)?;
fmri_to_metadata.get(key_bytes)?.is_some() // Check if the entry exists
}; let exists: bool = conn.query_row(
"SELECT EXISTS(SELECT 1 FROM obsoleted_packages WHERE fmri = ?1)",
rusqlite::params![&fmri],
|row| row.get(0),
)?;
// If the key doesn't exist in either table, return early if !exists {
if !exists_in_new_table {
return Ok(false); return Ok(false);
} }
// Now perform the actual removal // Remove the entry
let write_txn = self.db.begin_write()?; conn.execute(
{ "DELETE FROM obsoleted_packages WHERE fmri = ?1",
// Remove the entry from the new table rusqlite::params![&fmri],
if exists_in_new_table { )?;
let mut fmri_to_metadata = write_txn.open_table(FMRI_TO_METADATA_TABLE)?;
fmri_to_metadata.remove(key_bytes)?;
}
}
write_txn.commit()?;
Ok(true) Ok(true)
} }
@ -573,43 +479,61 @@ impl RedbObsoletedPackageIndex {
) -> Result<Option<(ObsoletedPackageMetadata, String)>> { ) -> Result<Option<(ObsoletedPackageMetadata, String)>> {
// Use the FMRI string directly as the key // Use the FMRI string directly as the key
let fmri = key.to_fmri_string(); let fmri = key.to_fmri_string();
let key_bytes = fmri.as_bytes();
// First, try to get the metadata directly from the new table let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
let metadata_result = {
let read_txn = self.db.begin_read()?;
let fmri_to_metadata = read_txn.open_table(FMRI_TO_METADATA_TABLE)?;
// Get the metadata bytes // Try to get the metadata from the database
match fmri_to_metadata.get(key_bytes)? { let metadata_result = match conn.query_row(
Some(bytes) => { "SELECT fmri, status, obsolescence_date, deprecation_message,
// Convert to owned bytes before the transaction is dropped obsoleted_by, metadata_version, content_hash
let metadata_bytes = bytes.value().to_vec(); FROM obsoleted_packages WHERE fmri = ?1",
// Try to deserialize the metadata rusqlite::params![&fmri],
match serde_cbor::from_slice::<ObsoletedPackageMetadata>(&metadata_bytes) { |row| {
Ok(metadata) => Some(metadata), Ok((
Err(e) => { row.get::<_, String>(0)?,
warn!( row.get::<_, String>(1)?,
"Failed to deserialize metadata from FMRI_TO_METADATA_TABLE with CBOR: {}", row.get::<_, String>(2)?,
e row.get::<_, Option<String>>(3)?,
); row.get::<_, Option<String>>(4)?,
None row.get::<_, u32>(5)?,
} row.get::<_, String>(6)?,
} ))
} },
None => None, ) {
} Ok(result) => Some(result),
Err(rusqlite::Error::QueryReturnedNoRows) => None,
Err(e) => return Err(e.into()),
}; };
// If we found the metadata in the new table, use it if let Some((
if let Some(metadata) = metadata_result { fmri,
// Get the content hash from the metadata status,
let content_hash = metadata.content_hash.clone(); obsolescence_date,
deprecation_message,
obsoleted_by_json,
metadata_version,
content_hash,
)) = metadata_result
{
// Deserialize obsoleted_by from JSON if present
let obsoleted_by = obsoleted_by_json
.as_ref()
.map(|json| serde_json::from_str::<Vec<String>>(json))
.transpose()?;
let metadata = ObsoletedPackageMetadata {
fmri,
status,
obsolescence_date,
deprecation_message,
obsoleted_by,
metadata_version,
content_hash: content_hash.clone(),
};
// For NULL_HASH entries, generate a minimal manifest // For NULL_HASH entries, generate a minimal manifest
let manifest_str = if content_hash == NULL_HASH { let manifest_str = if content_hash == NULL_HASH {
// Generate a minimal manifest for NULL_HASH entries // Generate a minimal manifest for NULL_HASH entries
// Construct an FMRI string from the metadata
format!( format!(
r#"{{ r#"{{
"attributes": [ "attributes": [
@ -631,13 +555,13 @@ impl RedbObsoletedPackageIndex {
) )
} else { } else {
// For non-NULL_HASH entries, get the manifest from the database // For non-NULL_HASH entries, get the manifest from the database
let read_txn = self.db.begin_read()?; match conn.query_row(
let hash_to_manifest = read_txn.open_table(HASH_TO_MANIFEST_TABLE)?; "SELECT manifest FROM obsoleted_manifests WHERE content_hash = ?1",
rusqlite::params![&content_hash],
// Get the manifest string |row| row.get::<_, String>(0),
match hash_to_manifest.get(content_hash.as_str())? { ) {
Some(manifest) => manifest.value().to_string(), Ok(manifest) => manifest,
None => { Err(rusqlite::Error::QueryReturnedNoRows) => {
warn!( warn!(
"Manifest not found for content hash: {}, generating minimal manifest", "Manifest not found for content hash: {}, generating minimal manifest",
content_hash content_hash
@ -663,6 +587,7 @@ impl RedbObsoletedPackageIndex {
metadata.fmri metadata.fmri
) )
} }
Err(e) => return Err(e.into()),
} }
}; };
Ok(Some((metadata, manifest_str))) Ok(Some((metadata, manifest_str)))
@ -676,33 +601,52 @@ impl RedbObsoletedPackageIndex {
&self, &self,
) -> Result<Vec<(ObsoletedPackageKey, ObsoletedPackageMetadata, String)>> { ) -> Result<Vec<(ObsoletedPackageKey, ObsoletedPackageMetadata, String)>> {
let mut entries = Vec::new(); let mut entries = Vec::new();
let mut processed_keys = std::collections::HashSet::new();
// First, collect all entries from the new table let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
{
let read_txn = self.db.begin_read()?;
let fmri_to_metadata = read_txn.open_table(FMRI_TO_METADATA_TABLE)?;
let mut iter = fmri_to_metadata.iter()?; let mut stmt = conn.prepare(
"SELECT fmri, publisher, stem, version, status, obsolescence_date,
deprecation_message, obsoleted_by, metadata_version, content_hash
FROM obsoleted_packages",
)?;
while let Some(entry) = iter.next() { let rows = stmt.query_map([], |row| {
let (key_bytes, metadata_bytes) = entry?; Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
row.get::<_, String>(3)?,
row.get::<_, String>(4)?,
row.get::<_, String>(5)?,
row.get::<_, Option<String>>(6)?,
row.get::<_, Option<String>>(7)?,
row.get::<_, u32>(8)?,
row.get::<_, String>(9)?,
))
})?;
// Convert to owned types before the transaction is dropped for row_result in rows {
let key_data = key_bytes.value().to_vec(); let (
let metadata_data = metadata_bytes.value().to_vec(); fmri,
_publisher,
// Convert key bytes to string and parse as FMRI _stem,
let fmri_str = match std::str::from_utf8(&key_data) { _version,
Ok(s) => s, status,
obsolescence_date,
deprecation_message,
obsoleted_by_json,
metadata_version,
content_hash,
) = match row_result {
Ok(r) => r,
Err(e) => { Err(e) => {
warn!("Failed to convert key bytes to string: {}", e); warn!("Failed to read row: {}", e);
continue; continue;
} }
}; };
// Parse the FMRI string to create an ObsoletedPackageKey // Parse the FMRI string to create an ObsoletedPackageKey
let key = match ObsoletedPackageKey::from_fmri_string(fmri_str) { let key = match ObsoletedPackageKey::from_fmri_string(&fmri) {
Ok(key) => key, Ok(key) => key,
Err(e) => { Err(e) => {
warn!("Failed to parse FMRI string: {}", e); warn!("Failed to parse FMRI string: {}", e);
@ -710,24 +654,28 @@ impl RedbObsoletedPackageIndex {
} }
}; };
let metadata: ObsoletedPackageMetadata = match serde_cbor::from_slice( // Deserialize obsoleted_by from JSON if present
&metadata_data, let obsoleted_by = match obsoleted_by_json
) { .as_ref()
Ok(metadata) => metadata, .map(|json| serde_json::from_str::<Vec<String>>(json))
.transpose()
{
Ok(obs) => obs,
Err(e) => { Err(e) => {
warn!( warn!("Failed to deserialize obsoleted_by JSON: {}", e);
"Failed to deserialize metadata from FMRI_TO_METADATA_TABLE with CBOR: {}",
e
);
continue; continue;
} }
}; };
// Add the key to the set of processed keys let metadata = ObsoletedPackageMetadata {
processed_keys.insert(key_data); fmri: fmri.clone(),
status,
// Get the content hash from the metadata obsolescence_date,
let content_hash = metadata.content_hash.clone(); deprecation_message,
obsoleted_by,
metadata_version,
content_hash: content_hash.clone(),
};
// For NULL_HASH entries, generate a minimal manifest // For NULL_HASH entries, generate a minimal manifest
let manifest_str = if content_hash == NULL_HASH { let manifest_str = if content_hash == NULL_HASH {
@ -749,16 +697,17 @@ impl RedbObsoletedPackageIndex {
}} }}
] ]
}}"#, }}"#,
metadata.fmri fmri
) )
} else { } else {
// For non-NULL_HASH entries, get the manifest from the database // For non-NULL_HASH entries, get the manifest from the database
let hash_to_manifest = read_txn.open_table(HASH_TO_MANIFEST_TABLE)?; match conn.query_row(
"SELECT manifest FROM obsoleted_manifests WHERE content_hash = ?1",
// Get the manifest string rusqlite::params![&content_hash],
match hash_to_manifest.get(content_hash.as_str())? { |row| row.get::<_, String>(0),
Some(manifest) => manifest.value().to_string(), ) {
None => { Ok(manifest) => manifest,
Err(rusqlite::Error::QueryReturnedNoRows) => {
warn!( warn!(
"Manifest not found for content hash: {}, generating minimal manifest", "Manifest not found for content hash: {}, generating minimal manifest",
content_hash content_hash
@ -781,15 +730,18 @@ impl RedbObsoletedPackageIndex {
}} }}
] ]
}}"#, }}"#,
metadata.fmri fmri
) )
} }
Err(e) => {
warn!("Failed to get manifest for content hash: {}", e);
continue;
}
} }
}; };
entries.push((key, metadata, manifest_str)); entries.push((key, metadata, manifest_str));
} }
}
Ok(entries) Ok(entries)
} }
@ -858,86 +810,26 @@ impl RedbObsoletedPackageIndex {
/// Clear the index /// Clear the index
fn clear(&self) -> Result<()> { fn clear(&self) -> Result<()> {
// Begin a writing transaction let conn = Connection::open(&self.db_path)?;
let write_txn = self.db.begin_write()?;
{
// Clear all tables by removing all entries
// Since redb doesn't have a clear() method, we need to iterate and remove each key
// Clear hash_to_manifest table // Clear both tables
{ conn.execute("DELETE FROM obsoleted_packages", [])?;
let mut hash_to_manifest = write_txn.open_table(FMRI_TO_METADATA_TABLE)?; conn.execute("DELETE FROM obsoleted_manifests", [])?;
let keys_to_remove = {
// First, collect all keys in a separate scope
let read_txn = self.db.begin_read()?;
let hash_to_manifest_read = read_txn.open_table(FMRI_TO_METADATA_TABLE)?;
let mut keys = Vec::new();
let mut iter = hash_to_manifest_read.iter()?;
while let Some(entry) = iter.next() {
let (key, _) = entry?;
keys.push(key.value().to_vec());
}
keys
};
// Then remove all keys
for key in keys_to_remove {
hash_to_manifest.remove(key.as_slice())?;
}
}
// Clear hash_to_manifest table
{
let mut hash_to_manifest = write_txn.open_table(HASH_TO_MANIFEST_TABLE)?;
let keys_to_remove = {
// First, collect all keys in a separate scope
let read_txn = self.db.begin_read()?;
let hash_to_manifest_read = read_txn.open_table(HASH_TO_MANIFEST_TABLE)?;
let mut keys = Vec::new();
let mut iter = hash_to_manifest_read.iter()?;
while let Some(entry) = iter.next() {
let (key, _) = entry?;
keys.push(key.value().to_string());
}
keys
};
// Then remove all keys
for key in keys_to_remove {
hash_to_manifest.remove(key.as_str())?;
}
}
}
write_txn.commit()?;
Ok(()) Ok(())
} }
/// Get the number of entries in the index /// Get the number of entries in the index
fn len(&self) -> Result<usize> { fn len(&self) -> Result<usize> {
// Begin a read transaction let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
let read_txn = self.db.begin_read()?;
// Open the fmri_to_hash table let count: i64 = conn.query_row(
let fmri_to_hash = read_txn.open_table(FMRI_TO_METADATA_TABLE)?; "SELECT COUNT(*) FROM obsoleted_packages",
[],
|row| row.get(0),
)?;
// Count the entries Ok(count as usize)
let mut count = 0;
let mut iter = fmri_to_hash.iter()?;
// Iterate through all entries and count them
while let Some(entry_result) = iter.next() {
// Just check if the entry exists, we don't need to access its value
entry_result?;
count += 1;
}
// Drop the iterator and table before returning
drop(iter);
drop(fmri_to_hash);
drop(read_txn);
Ok(count)
} }
/// Check if the index is empty /// Check if the index is empty
@ -1033,8 +925,8 @@ impl ObsoletedPackageMetadata {
pub struct ObsoletedPackageManager { pub struct ObsoletedPackageManager {
/// Base path for obsoleted packages /// Base path for obsoleted packages
base_path: PathBuf, base_path: PathBuf,
/// Index of obsoleted packages for faster lookups using redb /// Index of obsoleted packages for faster lookups using SQLite
index: RwLock<RedbObsoletedPackageIndex>, index: RwLock<SqliteObsoletedPackageIndex>,
} }
impl ObsoletedPackageManager { impl ObsoletedPackageManager {
@ -1075,14 +967,14 @@ impl ObsoletedPackageManager {
let base_path = repo_path.as_ref().join("obsoleted"); let base_path = repo_path.as_ref().join("obsoleted");
let index = { let index = {
// Create or open the redb-based index // Create or open the SQLite-based index
let redb_index = let sqlite_index =
RedbObsoletedPackageIndex::create_or_open(&base_path).unwrap_or_else(|e| { SqliteObsoletedPackageIndex::create_or_open(&base_path).unwrap_or_else(|e| {
// Log the error and create an empty redb index // Log the error and create an empty SQLite index
error!("Failed to create or open redb-based index: {}", e); error!("Failed to create or open SQLite-based index: {}", e);
RedbObsoletedPackageIndex::empty() SqliteObsoletedPackageIndex::empty()
}); });
RwLock::new(redb_index) RwLock::new(sqlite_index)
}; };
Self { base_path, index } Self { base_path, index }

164
libips/src/repository/shard_sync.rs Normal file
View file

@ -0,0 +1,164 @@
// This Source Code Form is subject to the terms of
// the Mozilla Public License, v. 2.0. If a copy of the
// MPL was not distributed with this file, You can
// obtain one at https://mozilla.org/MPL/2.0/.
//! Client-side shard synchronization.
//!
//! Downloads catalog shards from the repository server and verifies their integrity.
use crate::repository::sqlite_catalog::{ShardIndex, ShardEntry};
use miette::Diagnostic;
use sha2::{Digest, Sha256};
use std::fs;
use std::path::{Path, PathBuf};
use thiserror::Error;
#[derive(Debug, Error, Diagnostic)]
#[error("Shard sync error: {message}")]
#[diagnostic(code(ips::shard_sync_error))]
pub struct ShardSyncError {
pub message: String,
}
impl ShardSyncError {
fn new(msg: impl Into<String>) -> Self {
Self {
message: msg.into(),
}
}
}
impl From<reqwest::Error> for ShardSyncError {
fn from(e: reqwest::Error) -> Self {
Self::new(format!("HTTP error: {}", e))
}
}
impl From<std::io::Error> for ShardSyncError {
fn from(e: std::io::Error) -> Self {
Self::new(format!("IO error: {}", e))
}
}
impl From<serde_json::Error> for ShardSyncError {
fn from(e: serde_json::Error) -> Self {
Self::new(format!("JSON error: {}", e))
}
}
/// Synchronize catalog shards from a repository origin.
///
/// Downloads the shard index from `{origin_url}/{publisher}/catalog/2/catalog.attrs`,
/// compares hashes with local copies, and downloads only changed shards.
///
/// # Arguments
/// * `publisher` - Publisher name
/// * `origin_url` - Repository origin URL (e.g., "https://pkg.example.com")
/// * `local_shard_dir` - Local directory to store shards
/// * `download_obsolete` - Whether to download obsolete.db (default: false)
pub fn sync_shards(
publisher: &str,
origin_url: &str,
local_shard_dir: &Path,
download_obsolete: bool,
) -> Result<(), ShardSyncError> {
// Ensure local directory exists
fs::create_dir_all(local_shard_dir)?;
// Fetch shard index
let index_url = format!("{}/{}/catalog/2/catalog.attrs", origin_url, publisher);
let client = reqwest::blocking::Client::new();
let response = client.get(&index_url).send()?;
if !response.status().is_success() {
return Err(ShardSyncError::new(format!(
"Failed to fetch shard index: HTTP {}",
response.status()
)));
}
let index: ShardIndex = response.json()?;
// List of shards to sync
let shards_to_sync = if download_obsolete {
vec!["active.db", "fts.db", "obsolete.db"]
} else {
vec!["active.db", "fts.db"]
};
// Download each shard if needed
for shard_name in shards_to_sync {
let Some(shard_entry) = index.shards.get(shard_name) else {
tracing::warn!("Shard {} not found in index", shard_name);
continue;
};
let local_path = local_shard_dir.join(shard_name);
// Check if local copy exists and matches hash
let needs_download = if local_path.exists() {
match compute_sha256(&local_path) {
Ok(local_hash) => local_hash != shard_entry.sha256,
Err(_) => true, // Error reading local file, re-download
}
} else {
true
};
if !needs_download {
tracing::debug!("Shard {} is up to date", shard_name);
continue;
}
// Download shard
tracing::info!("Downloading shard {} from {}", shard_name, origin_url);
let shard_url = format!(
"{}/{}/catalog/2/{}",
origin_url, publisher, &shard_entry.sha256
);
let mut response = client.get(&shard_url).send()?;
if !response.status().is_success() {
return Err(ShardSyncError::new(format!(
"Failed to download shard {}: HTTP {}",
shard_name,
response.status()
)));
}
// Write to temporary file
let temp_path = local_shard_dir.join(format!("{}.tmp", shard_name));
let mut file = fs::File::create(&temp_path)?;
response.copy_to(&mut file)?;
drop(file);
// Verify SHA-256
let downloaded_hash = compute_sha256(&temp_path)?;
if downloaded_hash != shard_entry.sha256 {
fs::remove_file(&temp_path)?;
return Err(ShardSyncError::new(format!(
"SHA-256 mismatch for {}: expected {}, got {}",
shard_name, shard_entry.sha256, downloaded_hash
)));
}
// Atomic rename
fs::rename(&temp_path, &local_path)?;
tracing::info!("Successfully downloaded {}", shard_name);
}
// Write local copy of index for future comparisons
let index_json = serde_json::to_string_pretty(&index)?;
fs::write(local_shard_dir.join("catalog.attrs"), index_json)?;
Ok(())
}
/// Compute SHA-256 hash of a file.
fn compute_sha256(path: &Path) -> Result<String, ShardSyncError> {
let bytes = fs::read(path)?;
let mut hasher = Sha256::new();
hasher.update(&bytes);
Ok(format!("{:x}", hasher.finalize()))
}

484
libips/src/repository/sqlite_catalog.rs Normal file
View file

@ -0,0 +1,484 @@
// This Source Code Form is subject to the terms of
// the Mozilla Public License, v. 2.0. If a copy of the
// MPL was not distributed with this file, You can
// obtain one at https://mozilla.org/MPL/2.0/.
//! SQLite catalog shard generation and population.
//!
//! This module defines all SQLite schemas used by the IPS system and provides
//! functions to build pre-built catalog shards for distribution via the
//! catalog/2 endpoint.
use crate::actions::Manifest;
use crate::fmri::Fmri;
use crate::repository::catalog::CatalogManager;
use base64::Engine as _;
use miette::Diagnostic;
use rusqlite::Connection;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::BTreeMap;
use std::fs;
use std::path::Path;
use thiserror::Error;
/// Schema for active.db - contains non-obsolete packages and their dependencies.
/// No manifest blobs stored; manifests are fetched from repository on demand.
pub const ACTIVE_SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS packages (
stem TEXT NOT NULL,
version TEXT NOT NULL,
publisher TEXT NOT NULL,
fmri TEXT GENERATED ALWAYS AS (
'pkg://' || publisher || '/' || stem || '@' || version
) STORED,
PRIMARY KEY (stem, version, publisher)
);
CREATE INDEX IF NOT EXISTS idx_packages_fmri ON packages(fmri);
CREATE INDEX IF NOT EXISTS idx_packages_stem ON packages(stem);
CREATE TABLE IF NOT EXISTS dependencies (
pkg_stem TEXT NOT NULL,
pkg_version TEXT NOT NULL,
pkg_publisher TEXT NOT NULL,
dep_type TEXT NOT NULL,
dep_stem TEXT NOT NULL,
dep_version TEXT,
PRIMARY KEY (pkg_stem, pkg_version, pkg_publisher, dep_type, dep_stem)
);
CREATE INDEX IF NOT EXISTS idx_deps_pkg ON dependencies(pkg_stem, pkg_version, pkg_publisher);
CREATE TABLE IF NOT EXISTS incorporate_locks (
stem TEXT NOT NULL PRIMARY KEY,
release TEXT NOT NULL
);
"#;
/// Schema for obsolete.db - client-side shard for obsoleted packages.
pub const OBSOLETE_SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS obsolete_packages (
publisher TEXT NOT NULL,
stem TEXT NOT NULL,
version TEXT NOT NULL,
fmri TEXT GENERATED ALWAYS AS (
'pkg://' || publisher || '/' || stem || '@' || version
) STORED,
PRIMARY KEY (publisher, stem, version)
);
CREATE INDEX IF NOT EXISTS idx_obsolete_fmri ON obsolete_packages(fmri);
"#;
/// Schema for fts.db - full-text search index.
pub const FTS_SCHEMA: &str = r#"
CREATE VIRTUAL TABLE IF NOT EXISTS package_search
USING fts5(stem, publisher, summary, description,
content='', tokenize='unicode61');
"#;
/// Schema for installed.db - tracks installed packages with manifest blobs.
pub const INSTALLED_SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS installed (
fmri TEXT NOT NULL PRIMARY KEY,
manifest BLOB NOT NULL
);
"#;
/// Schema for index.db (repository/obsoleted.rs) - server-side obsoleted package index.
pub const OBSOLETED_INDEX_SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS obsoleted_packages (
fmri TEXT NOT NULL PRIMARY KEY,
publisher TEXT NOT NULL,
stem TEXT NOT NULL,
version TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'obsolete',
obsolescence_date TEXT NOT NULL,
deprecation_message TEXT,
obsoleted_by TEXT,
metadata_version INTEGER NOT NULL DEFAULT 1,
content_hash TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_obsidx_stem ON obsoleted_packages(stem);
CREATE TABLE IF NOT EXISTS obsoleted_manifests (
content_hash TEXT NOT NULL PRIMARY KEY,
manifest TEXT NOT NULL
);
"#;
#[derive(Debug, Error, Diagnostic)]
#[error("Shard building error: {message}")]
#[diagnostic(code(ips::shard_build_error))]
pub struct ShardBuildError {
pub message: String,
}
impl ShardBuildError {
fn new(msg: impl Into<String>) -> Self {
Self {
message: msg.into(),
}
}
}
impl From<rusqlite::Error> for ShardBuildError {
fn from(e: rusqlite::Error) -> Self {
Self::new(format!("SQLite error: {}", e))
}
}
impl From<std::io::Error> for ShardBuildError {
fn from(e: std::io::Error) -> Self {
Self::new(format!("IO error: {}", e))
}
}
impl From<serde_json::Error> for ShardBuildError {
fn from(e: serde_json::Error) -> Self {
Self::new(format!("JSON error: {}", e))
}
}
/// Shard metadata entry in catalog.attrs.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ShardEntry {
pub sha256: String,
pub size: u64,
#[serde(rename = "last-modified")]
pub last_modified: String,
}
/// Shard index JSON structure for catalog/2/catalog.attrs.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ShardIndex {
pub version: u32,
pub created: String,
#[serde(rename = "last-modified")]
pub last_modified: String,
#[serde(rename = "package-count")]
pub package_count: usize,
#[serde(rename = "package-version-count")]
pub package_version_count: usize,
pub shards: BTreeMap<String, ShardEntry>,
}
/// Build catalog shards from JSON catalog parts.
///
/// Reads catalog parts from `catalog_parts_dir`, generates active.db, obsolete.db,
/// and fts.db, writes them to `output_dir`, and creates catalog.attrs index.
pub fn build_shards(
catalog_parts_dir: &Path,
publisher: &str,
output_dir: &Path,
) -> Result<(), ShardBuildError> {
// Create temp directory for shard generation
fs::create_dir_all(output_dir)?;
let temp_dir = output_dir.join(".tmp");
fs::create_dir_all(&temp_dir)?;
// Create shard databases
let active_path = temp_dir.join("active.db");
let obsolete_path = temp_dir.join("obsolete.db");
let fts_path = temp_dir.join("fts.db");
let mut active_conn = Connection::open(&active_path)?;
let mut obsolete_conn = Connection::open(&obsolete_path)?;
let mut fts_conn = Connection::open(&fts_path)?;
// Execute schemas
active_conn.execute_batch(ACTIVE_SCHEMA)?;
obsolete_conn.execute_batch(OBSOLETE_SCHEMA)?;
fts_conn.execute_batch(FTS_SCHEMA)?;
// Read catalog parts
let catalog_manager = CatalogManager::new(catalog_parts_dir, publisher)
.map_err(|e| ShardBuildError::new(format!("Failed to create catalog manager: {}", e)))?;
let mut package_count = 0usize;
let mut package_version_count = 0usize;
// Begin transactions for batch inserts
let active_tx = active_conn.transaction()?;
let obsolete_tx = obsolete_conn.transaction()?;
let fts_tx = fts_conn.transaction()?;
{
let mut insert_pkg = active_tx.prepare(
"INSERT OR REPLACE INTO packages (stem, version, publisher) VALUES (?1, ?2, ?3)",
)?;
let mut insert_dep = active_tx.prepare(
"INSERT OR REPLACE INTO dependencies (pkg_stem, pkg_version, pkg_publisher, dep_type, dep_stem, dep_version) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
)?;
let mut insert_obs = obsolete_tx.prepare(
"INSERT OR REPLACE INTO obsolete_packages (publisher, stem, version) VALUES (?1, ?2, ?3)",
)?;
let mut insert_fts = fts_tx.prepare(
"INSERT INTO package_search (stem, publisher, summary, description) VALUES (?1, ?2, ?3, ?4)",
)?;
// Iterate catalog parts
let part_names: Vec<String> = catalog_manager.attrs().parts.keys().cloned().collect();
for part_name in part_names {
let part_path = catalog_parts_dir.join(&part_name);
// Load the CatalogPart
let part = crate::repository::catalog::CatalogPart::load(&part_path)
.map_err(|e| ShardBuildError::new(format!("Failed to load catalog part: {}", e)))?;
// Iterate through publishers in the catalog part
for (part_publisher, stems) in &part.packages {
// Only process packages for the requested publisher
if part_publisher != publisher {
continue;
}
// Iterate through package stems
for (pkg_name, versions) in stems {
// Iterate through versions
for version_entry in versions {
let pkg_version = &version_entry.version;
// Build a minimal manifest from the actions
let mut manifest = Manifest::new();
// Parse actions if available
if let Some(actions) = &version_entry.actions {
for action_str in actions {
if action_str.starts_with("set ") {
// Parse "set name=key value=val" format
let parts: Vec<&str> = action_str.split_whitespace().collect();
if parts.len() >= 3 {
if let Some(name_part) = parts.get(1) {
if let Some(key) = name_part.strip_prefix("name=") {
if let Some(value_part) = parts.get(2) {
if let Some(mut value) = value_part.strip_prefix("value=") {
// Remove quotes
value = value.trim_matches('"');
let mut attr = crate::actions::Attr::default();
attr.key = key.to_string();
attr.values = vec![value.to_string()];
manifest.attributes.push(attr);
}
}
}
}
}
} else if action_str.starts_with("depend ") {
// Parse "depend fmri=... type=..." format
let mut dep = crate::actions::Dependency::default();
for part in action_str.split_whitespace().skip(1) {
if let Some((k, v)) = part.split_once('=') {
match k {
"fmri" => {
if let Ok(f) = crate::fmri::Fmri::parse(v) {
dep.fmri = Some(f);
}
}
"type" => {
dep.dependency_type = v.to_string();
}
_ => {}
}
}
}
if dep.fmri.is_some() && !dep.dependency_type.is_empty() {
manifest.dependencies.push(dep);
}
}
}
}
// Determine if obsolete
let is_obsolete = crate::image::catalog::is_package_obsolete(&manifest);
// Count all package versions
package_version_count += 1;
// Obsolete packages go only to obsolete.db, non-obsolete go to active.db
if is_obsolete {
insert_obs.execute(rusqlite::params![publisher, pkg_name, pkg_version])?;
} else {
// Insert into packages table (active.db)
insert_pkg.execute(rusqlite::params![pkg_name, pkg_version, publisher])?;
// Extract and insert dependencies
for dep in &manifest.dependencies {
if dep.dependency_type == "require" || dep.dependency_type == "incorporate" {
if let Some(dep_fmri) = &dep.fmri {
let dep_stem = dep_fmri.stem();
let dep_version = dep_fmri.version.as_ref().map(|v| v.to_string());
insert_dep.execute(rusqlite::params![
pkg_name,
pkg_version,
publisher,
&dep.dependency_type,
dep_stem,
dep_version
])?;
}
}
}
}
// Extract summary and description for FTS
let summary = manifest
.attributes
.iter()
.find(|a| a.key == "pkg.summary")
.and_then(|a| a.values.first())
.map(|s| s.as_str())
.unwrap_or("");
let description = manifest
.attributes
.iter()
.find(|a| a.key == "pkg.description")
.and_then(|a| a.values.first())
.map(|s| s.as_str())
.unwrap_or("");
insert_fts.execute(rusqlite::params![
pkg_name,
publisher,
summary,
description
])?;
}
}
}
}
}
// Commit transactions
active_tx.commit()?;
obsolete_tx.commit()?;
fts_tx.commit()?;
// Count unique packages (stems)
let count: i64 = active_conn.query_row(
"SELECT COUNT(DISTINCT stem) FROM packages",
[],
|row| row.get(0),
)?;
package_count = count as usize;
// Close connections
drop(active_conn);
drop(obsolete_conn);
drop(fts_conn);
// Compute SHA-256 hashes and build index
let mut shards = BTreeMap::new();
let now = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string();
for (name, path) in [
("active.db", &active_path),
("obsolete.db", &obsolete_path),
("fts.db", &fts_path),
] {
let bytes = fs::read(path)?;
let mut hasher = Sha256::new();
hasher.update(&bytes);
let hash = format!("{:x}", hasher.finalize());
let size = bytes.len() as u64;
shards.insert(
name.to_string(),
ShardEntry {
sha256: hash.clone(),
size,
last_modified: now.clone(),
},
);
// Copy shard to output directory with both original name and hash-based name
// Keep original name for client-side use (e.g., active.db, obsolete.db)
let named_path = output_dir.join(name);
fs::copy(path, &named_path)?;
// Also copy to hash-based name for content-addressed server distribution
let hash_path = output_dir.join(&hash);
fs::copy(path, &hash_path)?;
}
// Write catalog.attrs
let index = ShardIndex {
version: 2,
created: now.clone(),
last_modified: now,
package_count,
package_version_count,
shards,
};
let index_json = serde_json::to_string_pretty(&index)?;
fs::write(output_dir.join("catalog.attrs"), index_json)?;
// Clean up temp directory
fs::remove_dir_all(&temp_dir).ok();
Ok(())
}
/// Helper function for tests: populate active.db with a single package.
/// Creates tables if absent (idempotent).
pub fn populate_active_db(
db_path: &Path,
fmri: &Fmri,
manifest: &Manifest,
) -> Result<(), ShardBuildError> {
let mut conn = Connection::open(db_path)?;
conn.execute_batch(ACTIVE_SCHEMA)?;
let tx = conn.transaction()?;
{
tx.execute(
"INSERT OR REPLACE INTO packages (stem, version, publisher) VALUES (?1, ?2, ?3)",
rusqlite::params![
fmri.stem(),
fmri.version(),
fmri.publisher.as_deref().unwrap_or("")
],
)?;
for dep in &manifest.dependencies {
if dep.dependency_type == "require" || dep.dependency_type == "incorporate" {
if let Some(dep_fmri) = &dep.fmri {
let dep_stem = dep_fmri.stem();
let dep_version = dep_fmri.version.as_ref().map(|v| v.to_string());
tx.execute(
"INSERT OR REPLACE INTO dependencies (pkg_stem, pkg_version, pkg_publisher, dep_type, dep_stem, dep_version) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
rusqlite::params![
fmri.stem(),
fmri.version(),
fmri.publisher.as_deref().unwrap_or(""),
&dep.dependency_type,
dep_stem,
dep_version
],
)?;
}
}
}
}
tx.commit()?;
Ok(())
}
/// Helper function for tests: mark a package as obsolete in obsolete.db.
/// Creates tables if absent (idempotent).
pub fn populate_obsolete_db(db_path: &Path, fmri: &Fmri) -> Result<(), ShardBuildError> {
let mut conn = Connection::open(db_path)?;
conn.execute_batch(OBSOLETE_SCHEMA)?;
conn.execute(
"INSERT OR REPLACE INTO obsolete_packages (publisher, stem, version) VALUES (?1, ?2, ?3)",
rusqlite::params![
fmri.publisher.as_deref().unwrap_or(""),
fmri.stem(),
fmri.version()
],
)?;
Ok(())
}
// Note: compress_json_lz4, decode_manifest_bytes, and is_package_obsolete
// are available as pub(crate) in crate::image::catalog and can be used
// within libips but not re-exported.

365
libips/src/solver/mod.rs
View file

@ -18,55 +18,23 @@
use miette::Diagnostic; use miette::Diagnostic;
// Begin resolvo wiring imports (names discovered by compiler) // Begin resolvo wiring imports (names discovered by compiler)
// We start broad and refine with compiler guidance. // We start broad and refine with compiler guidance.
use lz4::Decoder as Lz4Decoder;
use redb::{ReadableDatabase, ReadableTable};
use resolvo::{ use resolvo::{
self, Candidates, Condition, ConditionId, ConditionalRequirement, self, Candidates, Condition, ConditionId, ConditionalRequirement,
Dependencies as RDependencies, DependencyProvider, HintDependenciesAvailable, Interner, Dependencies as RDependencies, DependencyProvider, HintDependenciesAvailable, Interner,
KnownDependencies, Mapping, NameId, Problem as RProblem, SolvableId, Solver as RSolver, KnownDependencies, Mapping, NameId, Problem as RProblem, SolvableId, Solver as RSolver,
SolverCache, StringId, UnsolvableOrCancelled, VersionSetId, VersionSetUnionId, SolverCache, StringId, UnsolvableOrCancelled, VersionSetId, VersionSetUnionId,
}; };
use rusqlite::{Connection, OpenFlags};
use std::cell::RefCell; use std::cell::RefCell;
use std::collections::{BTreeMap, HashMap}; use std::collections::{BTreeMap, HashMap};
use std::fmt::Display; use std::fmt::Display;
use std::io::{Cursor, Read};
use thiserror::Error; use thiserror::Error;
use crate::actions::Manifest; use crate::actions::Manifest;
use crate::image::catalog::{CATALOG_TABLE, INCORPORATE_TABLE};
// Public advice API lives in a sibling module // Public advice API lives in a sibling module
pub mod advice; pub mod advice;
// Local helpers to decode manifest bytes stored in catalog DB (JSON or LZ4-compressed JSON)
fn is_likely_json_local(bytes: &[u8]) -> bool {
let mut i = 0;
while i < bytes.len() && matches!(bytes[i], b' ' | b'\n' | b'\r' | b'\t') {
i += 1;
}
if i >= bytes.len() {
return false;
}
matches!(bytes[i], b'{' | b'[')
}
fn decode_manifest_bytes_local(bytes: &[u8]) -> Result<Manifest, serde_json::Error> {
if is_likely_json_local(bytes) {
return serde_json::from_slice::<Manifest>(bytes);
}
// Try LZ4; on failure, fall back to JSON attempt
if let Ok(mut dec) = Lz4Decoder::new(Cursor::new(bytes)) {
let mut out = Vec::new();
if dec.read_to_end(&mut out).is_ok() {
if let Ok(m) = serde_json::from_slice::<Manifest>(&out) {
return Ok(m);
}
}
}
// Fallback to JSON parse of original bytes
serde_json::from_slice::<Manifest>(bytes)
}
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
struct PkgCand { struct PkgCand {
#[allow(dead_code)] #[allow(dead_code)]
@ -85,11 +53,8 @@ enum VersionSetKind {
struct IpsProvider<'a> { struct IpsProvider<'a> {
image: &'a Image, image: &'a Image,
// Persistent database handles and read transactions for catalog/obsoleted // SQLite connection to active.db (catalog), opened read-only
_catalog_db: redb::Database, catalog_conn: Connection,
catalog_tx: redb::ReadTransaction,
_obsoleted_db: redb::Database,
_obsoleted_tx: redb::ReadTransaction,
// interner storages // interner storages
names: Mapping<NameId, String>, names: Mapping<NameId, String>,
name_by_str: BTreeMap<String, NameId>, name_by_str: BTreeMap<String, NameId>,
@ -108,24 +73,21 @@ use crate::image::Image;
impl<'a> IpsProvider<'a> { impl<'a> IpsProvider<'a> {
fn new(image: &'a Image) -> Result<Self, SolverError> { fn new(image: &'a Image) -> Result<Self, SolverError> {
// Open databases and keep read transactions alive for the provider lifetime // Open active.db (catalog) read-only with WAL mode for better concurrency
let catalog_db = redb::Database::open(image.catalog_db_path()) let catalog_conn = Connection::open_with_flags(
&image.active_db_path(),
OpenFlags::SQLITE_OPEN_READ_ONLY,
)
.map_err(|e| SolverError::new(format!("open catalog db: {}", e)))?; .map_err(|e| SolverError::new(format!("open catalog db: {}", e)))?;
let catalog_tx = catalog_db
.begin_read() // Enable WAL mode for better concurrency (ignored if already set)
.map_err(|e| SolverError::new(format!("begin read catalog db: {}", e)))?; catalog_conn
let obsoleted_db = redb::Database::open(image.obsoleted_db_path()) .pragma_update(None, "journal_mode", "WAL")
.map_err(|e| SolverError::new(format!("open obsoleted db: {}", e)))?; .ok();
let obsoleted_tx = obsoleted_db
.begin_read()
.map_err(|e| SolverError::new(format!("begin read obsoleted db: {}", e)))?;
let mut prov = IpsProvider { let mut prov = IpsProvider {
image, image,
_catalog_db: catalog_db, catalog_conn,
catalog_tx,
_obsoleted_db: obsoleted_db,
_obsoleted_tx: obsoleted_tx,
names: Mapping::default(), names: Mapping::default(),
name_by_str: BTreeMap::new(), name_by_str: BTreeMap::new(),
strings: Mapping::default(), strings: Mapping::default(),
@ -141,30 +103,52 @@ impl<'a> IpsProvider<'a> {
} }
fn build_index(&mut self) -> Result<(), SolverError> { fn build_index(&mut self) -> Result<(), SolverError> {
use crate::image::catalog::CATALOG_TABLE; // Query packages table directly - no manifest decoding needed
// Iterate catalog table and build in-memory index of non-obsolete candidates // Use a scope to ensure stmt is dropped before we start mutating self
let table = self let collected_rows: Vec<Result<(String, String, String), rusqlite::Error>> = {
.catalog_tx let mut stmt = self
.open_table(CATALOG_TABLE) .catalog_conn
.map_err(|e| SolverError::new(format!("open catalog table: {}", e)))?; .prepare("SELECT stem, version, publisher FROM packages")
.map_err(|e| SolverError::new(format!("prepare packages query: {}", e)))?;
let rows = stmt
.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
))
})
.map_err(|e| SolverError::new(format!("query packages: {}", e)))?;
// Collect all rows into a Vec to avoid holding a borrow on the connection
rows.collect()
};
// Temporary map: stem string -> Vec<Fmri> // Temporary map: stem string -> Vec<Fmri>
let mut by_stem: BTreeMap<String, Vec<Fmri>> = BTreeMap::new(); let mut by_stem: BTreeMap<String, Vec<Fmri>> = BTreeMap::new();
for entry in table
.iter()
.map_err(|e| SolverError::new(format!("iterate catalog table: {}", e)))?
{
let (k, v) =
entry.map_err(|e| SolverError::new(format!("read catalog entry: {}", e)))?;
let key = k.value(); // stem@version
// Try to decode manifest and extract full FMRI (including publisher) for row_result in collected_rows {
let mut pushed = false; let (stem, version, publisher) = row_result
if let Ok(manifest) = decode_manifest_bytes_local(v.value()) { .map_err(|e| SolverError::new(format!("read package row: {}", e)))?;
if let Some(attr) = manifest.attributes.iter().find(|a| a.key == "pkg.fmri") {
if let Some(fmri_str) = attr.values.first() { // Parse version
if let Ok(mut fmri) = Fmri::parse(fmri_str) { let ver_obj = crate::fmri::Version::parse(&version).ok();
// Ensure publisher is present; if missing/empty, use image default publisher
// Build FMRI with publisher, stem, and version
let mut fmri = if let Some(v) = ver_obj {
Fmri::with_publisher(&publisher, &stem, Some(v))
} else {
// No parsable version; still record a minimal FMRI without version
Fmri::with_publisher(&publisher, &stem, None)
};
// Normalize: empty publisher string -> None
if fmri.publisher.as_deref() == Some("") {
fmri.publisher = None;
}
// If publisher is still None/empty, try using image default publisher
let missing_pub = fmri let missing_pub = fmri
.publisher .publisher
.as_deref() .as_deref()
@ -175,46 +159,14 @@ impl<'a> IpsProvider<'a> {
fmri.publisher = Some(defp.name.clone()); fmri.publisher = Some(defp.name.clone());
} }
} }
by_stem
.entry(fmri.stem().to_string())
.or_default()
.push(fmri);
pushed = true;
}
}
}
}
// Fallback: derive FMRI from catalog key if we couldn't push from manifest by_stem.entry(stem).or_default().push(fmri);
if !pushed {
if let Some((stem, ver_str)) = key.split_once('@') {
let ver_obj = crate::fmri::Version::parse(ver_str).ok();
// Prefer default publisher if configured; else leave None by constructing and then setting publisher
let mut fmri = if let Some(v) = ver_obj.clone() {
if let Ok(defp) = self.image.default_publisher() {
Fmri::with_publisher(&defp.name, stem, Some(v))
} else {
Fmri::with_version(stem, v)
}
} else {
// No parsable version; still record a minimal FMRI without version
if let Ok(defp) = self.image.default_publisher() {
Fmri::with_publisher(&defp.name, stem, None)
} else {
Fmri::with_publisher("", stem, None)
}
};
// Normalize: empty publisher string -> None
if fmri.publisher.as_deref() == Some("") {
fmri.publisher = None;
}
by_stem.entry(stem.to_string()).or_default().push(fmri);
}
}
} }
// Intern and populate solvables per stem // Intern and populate solvables per stem
for (stem, mut fmris) in by_stem { // Collect into Vec to avoid borrow checker issues with mutating self while iterating
let stems_and_fmris: Vec<(String, Vec<Fmri>)> = by_stem.into_iter().collect();
for (stem, mut fmris) in stems_and_fmris {
let name_id = self.intern_name(&stem); let name_id = self.intern_name(&stem);
// Sort fmris newest-first using IPS ordering // Sort fmris newest-first using IPS ordering
fmris.sort_by(|a, b| version_order_desc(a, b)); fmris.sort_by(|a, b| version_order_desc(a, b));
@ -254,22 +206,13 @@ impl<'a> IpsProvider<'a> {
} }
fn lookup_incorporated_release(&self, stem: &str) -> Option<String> { fn lookup_incorporated_release(&self, stem: &str) -> Option<String> {
if let Ok(table) = self.catalog_tx.open_table(INCORPORATE_TABLE) { self.catalog_conn
if let Ok(Some(rel)) = table.get(stem) { .query_row(
return Some(String::from_utf8_lossy(rel.value()).to_string()); "SELECT release FROM incorporate_locks WHERE stem = ?1",
} rusqlite::params![stem],
} |row| row.get(0),
None )
} .ok()
fn read_manifest_from_catalog(&self, fmri: &Fmri) -> Option<Manifest> {
let key = format!("{}@{}", fmri.stem(), fmri.version());
if let Ok(table) = self.catalog_tx.open_table(CATALOG_TABLE) {
if let Ok(Some(bytes)) = table.get(key.as_str()) {
return decode_manifest_bytes_local(bytes.value()).ok();
}
}
None
} }
} }
@ -516,9 +459,31 @@ impl<'a> DependencyProvider for IpsProvider<'a> {
async fn get_dependencies(&self, solvable: SolvableId) -> RDependencies { async fn get_dependencies(&self, solvable: SolvableId) -> RDependencies {
let pkg = self.solvables.get(solvable).unwrap(); let pkg = self.solvables.get(solvable).unwrap();
let fmri = &pkg.fmri; let fmri = &pkg.fmri;
let manifest_opt = self.read_manifest_from_catalog(fmri);
let Some(manifest) = manifest_opt else { // Query dependencies table directly instead of decoding manifest
return RDependencies::Known(KnownDependencies::default()); let mut stmt = match self.catalog_conn.prepare(
"SELECT dep_stem, dep_version FROM dependencies
WHERE pkg_stem = ?1 AND pkg_version = ?2 AND pkg_publisher = ?3 AND dep_type = 'require'"
) {
Ok(s) => s,
Err(_) => return RDependencies::Known(KnownDependencies::default()),
};
let parent_stem = fmri.stem();
let parent_version = fmri.version();
let parent_publisher = fmri.publisher.as_deref().unwrap_or("");
let rows = match stmt.query_map(
rusqlite::params![parent_stem, parent_version, parent_publisher],
|row| {
Ok((
row.get::<_, String>(0)?, // dep_stem
row.get::<_, Option<String>>(1)?, // dep_version
))
},
) {
Ok(r) => r,
Err(_) => return RDependencies::Known(KnownDependencies::default()),
}; };
// Build requirements for "require" deps // Build requirements for "require" deps
@ -526,19 +491,24 @@ impl<'a> DependencyProvider for IpsProvider<'a> {
let parent_branch = fmri.version.as_ref().and_then(|v| v.branch.clone()); let parent_branch = fmri.version.as_ref().and_then(|v| v.branch.clone());
let parent_pub = fmri.publisher.as_deref(); let parent_pub = fmri.publisher.as_deref();
for d in manifest for row_result in rows {
.dependencies let (dep_stem, dep_version_str) = match row_result {
.iter() Ok(r) => r,
.filter(|d| d.dependency_type == "require") Err(_) => continue,
{ };
if let Some(df) = &d.fmri {
let stem = df.stem().to_string(); let Some(child_name_id) = self.name_by_str.get(&dep_stem).copied() else {
let Some(child_name_id) = self.name_by_str.get(&stem).copied() else {
// If the dependency name isn't present in the catalog index, skip it // If the dependency name isn't present in the catalog index, skip it
continue; continue;
}; };
// Parse dep_version to extract release component
let dep_version = dep_version_str
.as_ref()
.and_then(|s| crate::fmri::Version::parse(s).ok());
// Create version set by release (from dep expr) and branch (from parent) // Create version set by release (from dep expr) and branch (from parent)
let vs_kind = match (&df.version, &parent_branch) { let vs_kind = match (&dep_version, &parent_branch) {
(Some(ver), Some(branch)) => VersionSetKind::ReleaseAndBranch { (Some(ver), Some(branch)) => VersionSetKind::ReleaseAndBranch {
release: ver.release.clone(), release: ver.release.clone(),
branch: branch.clone(), branch: branch.clone(),
@ -557,7 +527,7 @@ impl<'a> DependencyProvider for IpsProvider<'a> {
.entry(child_name_id) .entry(child_name_id)
.or_insert(order); .or_insert(order);
} }
}
RDependencies::Known(KnownDependencies { RDependencies::Known(KnownDependencies {
requirements: reqs, requirements: reqs,
constrains: vec![], constrains: vec![],
@ -820,18 +790,6 @@ pub fn resolve_install(
} }
name_to_fmris.insert(*name_id, v); name_to_fmris.insert(*name_id, v);
} }
// Snapshot: Catalog manifest cache keyed by stem@version for all candidates
let mut key_to_manifest: HashMap<String, Manifest> = HashMap::new();
for fmris in name_to_fmris.values() {
for fmri in fmris {
let key = format!("{}@{}", fmri.stem(), fmri.version());
if !key_to_manifest.contains_key(&key) {
if let Some(man) = provider.read_manifest_from_catalog(fmri) {
key_to_manifest.insert(key, man);
}
}
}
}
// Run the solver // Run the solver
let roots_for_err: Vec<Constraint> = root_names.iter().map(|(_, c)| c.clone()).collect(); let roots_for_err: Vec<Constraint> = root_names.iter().map(|(_, c)| c.clone()).collect();
@ -864,15 +822,10 @@ pub fn resolve_install(
let mut plan = InstallPlan::default(); let mut plan = InstallPlan::default();
for sid in solution_ids { for sid in solution_ids {
if let Some(fmri) = sid_to_fmri.get(&sid).cloned() { if let Some(fmri) = sid_to_fmri.get(&sid).cloned() {
// Prefer repository manifest; fallback to preloaded catalog snapshot, then image catalog // Fetch manifest from repository or catalog cache
let key = format!("{}@{}", fmri.stem(), fmri.version());
let manifest = match image_ref.get_manifest_from_repository(&fmri) { let manifest = match image_ref.get_manifest_from_repository(&fmri) {
Ok(m) => m, Ok(m) => m,
Err(repo_err) => { Err(repo_err) => match image_ref.get_manifest_from_catalog(&fmri) {
if let Some(m) = key_to_manifest.get(&key).cloned() {
m
} else {
match image_ref.get_manifest_from_catalog(&fmri) {
Ok(Some(m)) => m, Ok(Some(m)) => m,
_ => { _ => {
return Err(SolverError::new(format!( return Err(SolverError::new(format!(
@ -880,9 +833,7 @@ pub fn resolve_install(
fmri, repo_err fmri, repo_err
))); )));
} }
} },
}
}
}; };
plan.reasons.push(format!("selected {} via solver", fmri)); plan.reasons.push(format!("selected {} via solver", fmri));
plan.add.push(ResolvedPkg { fmri, manifest }); plan.add.push(ResolvedPkg { fmri, manifest });
@ -933,8 +884,7 @@ mod solver_integration_tests {
use crate::actions::Dependency; use crate::actions::Dependency;
use crate::fmri::Version; use crate::fmri::Version;
use crate::image::ImageType; use crate::image::ImageType;
use crate::image::catalog::{CATALOG_TABLE, OBSOLETED_TABLE}; use crate::repository::sqlite_catalog;
use redb::Database;
use tempfile::tempdir; use tempfile::tempdir;
fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version { fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version {
@ -970,34 +920,13 @@ mod solver_integration_tests {
} }
fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) { fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) {
let db = Database::open(image.catalog_db_path()).expect("open catalog db"); sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest)
let tx = db.begin_write().expect("begin write"); .expect("populate active db");
{
let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table");
let key = format!("{}@{}", fmri.stem(), fmri.version());
let val = serde_json::to_vec(manifest).expect("serialize manifest");
table
.insert(key.as_str(), val.as_slice())
.expect("insert manifest");
}
tx.commit().expect("commit");
} }
fn mark_obsolete(image: &Image, fmri: &Fmri) { fn mark_obsolete(image: &Image, fmri: &Fmri) {
let db = Database::open(image.obsoleted_db_path()).expect("open obsoleted db"); sqlite_catalog::populate_obsolete_db(&image.obsolete_db_path(), fmri)
let tx = db.begin_write().expect("begin write"); .expect("populate obsolete db");
{
let mut table = tx
.open_table(OBSOLETED_TABLE)
.expect("open obsoleted table");
let key = fmri.to_string();
// store empty value
let empty: Vec<u8> = Vec::new();
table
.insert(key.as_str(), empty.as_slice())
.expect("insert obsolete");
}
tx.commit().expect("commit");
} }
fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image { fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image {
@ -1322,8 +1251,7 @@ mod solver_error_message_tests {
use crate::actions::{Dependency, Manifest}; use crate::actions::{Dependency, Manifest};
use crate::fmri::{Fmri, Version}; use crate::fmri::{Fmri, Version};
use crate::image::ImageType; use crate::image::ImageType;
use crate::image::catalog::CATALOG_TABLE; use crate::repository::sqlite_catalog;
use redb::Database;
fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version { fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version {
let mut v = Version::new(release); let mut v = Version::new(release);
@ -1354,17 +1282,8 @@ mod solver_error_message_tests {
} }
fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) { fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) {
let db = Database::open(image.catalog_db_path()).expect("open catalog db"); sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest)
let tx = db.begin_write().expect("begin write"); .expect("populate active db");
{
let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table");
let key = format!("{}@{}", fmri.stem(), fmri.version());
let val = serde_json::to_vec(manifest).expect("serialize manifest");
table
.insert(key.as_str(), val.as_slice())
.expect("insert manifest");
}
tx.commit().expect("commit");
} }
#[test] #[test]
@ -1427,8 +1346,7 @@ mod incorporate_lock_tests {
use crate::actions::Dependency; use crate::actions::Dependency;
use crate::fmri::Version; use crate::fmri::Version;
use crate::image::ImageType; use crate::image::ImageType;
use crate::image::catalog::CATALOG_TABLE; use crate::repository::sqlite_catalog;
use redb::Database;
use tempfile::tempdir; use tempfile::tempdir;
fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version { fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version {
@ -1447,17 +1365,8 @@ mod incorporate_lock_tests {
} }
fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) { fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) {
let db = Database::open(image.catalog_db_path()).expect("open catalog db"); sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest)
let tx = db.begin_write().expect("begin write"); .expect("populate active db");
{
let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table");
let key = format!("{}@{}", fmri.stem(), fmri.version());
let val = serde_json::to_vec(manifest).expect("serialize manifest");
table
.insert(key.as_str(), val.as_slice())
.expect("insert manifest");
}
tx.commit().expect("commit");
} }
fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image { fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image {
@ -1638,8 +1547,7 @@ mod composite_release_tests {
use crate::actions::{Dependency, Manifest}; use crate::actions::{Dependency, Manifest};
use crate::fmri::{Fmri, Version}; use crate::fmri::{Fmri, Version};
use crate::image::ImageType; use crate::image::ImageType;
use crate::image::catalog::CATALOG_TABLE; use crate::repository::sqlite_catalog;
use redb::Database;
fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version { fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version {
let mut v = Version::new(release); let mut v = Version::new(release);
@ -1657,17 +1565,8 @@ mod composite_release_tests {
} }
fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) { fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) {
let db = Database::open(image.catalog_db_path()).expect("open catalog db"); sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest)
let tx = db.begin_write().expect("begin write"); .expect("populate active db");
{
let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table");
let key = format!("{}@{}", fmri.stem(), fmri.version());
let val = serde_json::to_vec(manifest).expect("serialize manifest");
table
.insert(key.as_str(), val.as_slice())
.expect("insert manifest");
}
tx.commit().expect("commit");
} }
fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image { fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image {
@ -1772,8 +1671,7 @@ mod circular_dependency_tests {
use crate::actions::Dependency; use crate::actions::Dependency;
use crate::fmri::{Fmri, Version}; use crate::fmri::{Fmri, Version};
use crate::image::ImageType; use crate::image::ImageType;
use crate::image::catalog::CATALOG_TABLE; use crate::repository::sqlite_catalog;
use redb::Database;
use std::collections::HashSet; use std::collections::HashSet;
fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version { fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version {
@ -1809,17 +1707,8 @@ mod circular_dependency_tests {
} }
fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) { fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) {
let db = Database::open(image.catalog_db_path()).expect("open catalog db"); sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest)
let tx = db.begin_write().expect("begin write"); .expect("populate active db");
{
let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table");
let key = format!("{}@{}", fmri.stem(), fmri.version());
let val = serde_json::to_vec(manifest).expect("serialize manifest");
table
.insert(key.as_str(), val.as_slice())
.expect("insert manifest");
}
tx.commit().expect("commit");
} }
fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image { fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image {

6
pkg6depotd/Cargo.toml
View file

@ -31,6 +31,8 @@ serde_json = "1.0"
dirs = "6" dirs = "6"
nix = { version = "0.30", features = ["signal", "process", "user", "fs"] } nix = { version = "0.30", features = ["signal", "process", "user", "fs"] }
sha1 = "0.10" sha1 = "0.10"
sha2 = "0.10"
rusqlite = { version = "0.31", default-features = false }
chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
flate2 = "1" flate2 = "1"
httpdate = "1" httpdate = "1"
@ -55,3 +57,7 @@ reqwest = { version = "0.12", features = ["blocking", "json"] }
assert_cmd = "2" assert_cmd = "2"
predicates = "3" predicates = "3"
tempfile = "3" tempfile = "3"
[features]
default = ["bundled-sqlite"]
bundled-sqlite = ["rusqlite/bundled"]

1
pkg6depotd/src/http/handlers/mod.rs
View file

@ -4,4 +4,5 @@ pub mod info;
pub mod manifest; pub mod manifest;
pub mod publisher; pub mod publisher;
pub mod search; pub mod search;
pub mod shard;
pub mod versions; pub mod versions;

127
pkg6depotd/src/http/handlers/shard.rs Normal file
View file

@ -0,0 +1,127 @@
use crate::errors::DepotError;
use crate::repo::DepotRepo;
use axum::extract::{Path, Request, State};
use axum::http::header;
use axum::response::{IntoResponse, Response};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::fs;
use std::sync::Arc;
use tower::ServiceExt;
use tower_http::services::ServeFile;
/// Shard metadata entry in catalog.attrs.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct ShardEntry {
sha256: String,
size: u64,
#[serde(rename = "last-modified")]
last_modified: String,
}
/// Shard index JSON structure for catalog/2/catalog.attrs.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct ShardIndex {
version: u32,
created: String,
#[serde(rename = "last-modified")]
last_modified: String,
#[serde(rename = "package-count")]
package_count: usize,
#[serde(rename = "package-version-count")]
package_version_count: usize,
shards: BTreeMap<String, ShardEntry>,
}
/// GET /{publisher}/catalog/2/catalog.attrs
pub async fn get_shard_index(
State(repo): State<Arc<DepotRepo>>,
Path(publisher): Path<String>,
) -> Result<Response, DepotError> {
let shard_dir = repo.shard_dir(&publisher);
let index_path = shard_dir.join("catalog.attrs");
if !index_path.exists() {
return Err(DepotError::Repo(
libips::repository::RepositoryError::NotFound(
"catalog.attrs not found - shards not yet built".to_string(),
),
));
}
let content = fs::read_to_string(&index_path)
.map_err(|e| DepotError::Server(format!("Failed to read catalog.attrs: {}", e)))?;
Ok(([(header::CONTENT_TYPE, "application/json")], content).into_response())
}
/// GET /{publisher}/catalog/2/{sha256}
pub async fn get_shard_blob(
State(repo): State<Arc<DepotRepo>>,
Path((publisher, sha256)): Path<(String, String)>,
req: Request,
) -> Result<Response, DepotError> {
let shard_dir = repo.shard_dir(&publisher);
let index_path = shard_dir.join("catalog.attrs");
if !index_path.exists() {
return Err(DepotError::Repo(
libips::repository::RepositoryError::NotFound(
"catalog.attrs not found - shards not yet built".to_string(),
),
));
}
// Read index to validate hash
let index_content = fs::read_to_string(&index_path)
.map_err(|e| DepotError::Server(format!("Failed to read catalog.attrs: {}", e)))?;
let index: ShardIndex = serde_json::from_str(&index_content)
.map_err(|e| DepotError::Server(format!("Failed to parse catalog.attrs: {}", e)))?;
// Find which shard file corresponds to this hash
let mut shard_path: Option<std::path::PathBuf> = None;
for (name, entry) in &index.shards {
if entry.sha256 == sha256 {
shard_path = Some(shard_dir.join(&sha256));
break;
}
}
let Some(path) = shard_path else {
return Err(DepotError::Repo(
libips::repository::RepositoryError::NotFound(format!(
"Shard with hash {} not found",
sha256
)),
));
};
if !path.exists() {
return Err(DepotError::Repo(
libips::repository::RepositoryError::NotFound(format!(
"Shard file {} not found on disk",
sha256
)),
));
}
// Serve the file
let service = ServeFile::new(path);
let result = service.oneshot(req).await;
match result {
Ok(mut res) => {
// Add cache headers - content is content-addressed and immutable
res.headers_mut().insert(
header::CONTENT_TYPE,
header::HeaderValue::from_static("application/octet-stream"),
);
res.headers_mut().insert(
header::CACHE_CONTROL,
header::HeaderValue::from_static("public, immutable, max-age=86400"),
);
Ok(res.into_response())
}
Err(e) => Err(DepotError::Server(e.to_string())),
}
}

2
pkg6depotd/src/http/handlers/versions.rs
View file

@ -68,7 +68,7 @@ pub async fn get_versions() -> impl IntoResponse {
}, },
SupportedOperation { SupportedOperation {
op: Operation::Catalog, op: Operation::Catalog,
versions: vec![1], versions: vec![1, 2],
}, },
SupportedOperation { SupportedOperation {
op: Operation::Manifest, op: Operation::Manifest,

10
pkg6depotd/src/http/routes.rs
View file

@ -1,5 +1,5 @@
use crate::http::admin; use crate::http::admin;
use crate::http::handlers::{catalog, file, info, manifest, publisher, search, versions}; use crate::http::handlers::{catalog, file, info, manifest, publisher, search, shard, versions};
use crate::repo::DepotRepo; use crate::repo::DepotRepo;
use axum::{ use axum::{
Router, Router,
@ -16,6 +16,14 @@ pub fn app_router(state: Arc<DepotRepo>) -> Router {
"/{publisher}/catalog/1/{filename}", "/{publisher}/catalog/1/{filename}",
get(catalog::get_catalog_v1).head(catalog::get_catalog_v1), get(catalog::get_catalog_v1).head(catalog::get_catalog_v1),
) )
.route(
"/{publisher}/catalog/2/catalog.attrs",
get(shard::get_shard_index).head(shard::get_shard_index),
)
.route(
"/{publisher}/catalog/2/{sha256}",
get(shard::get_shard_blob).head(shard::get_shard_blob),
)
.route( .route(
"/{publisher}/manifest/0/{fmri}", "/{publisher}/manifest/0/{fmri}",
get(manifest::get_manifest).head(manifest::get_manifest), get(manifest::get_manifest).head(manifest::get_manifest),

7
pkg6depotd/src/repo.rs
View file

@ -104,6 +104,13 @@ impl DepotRepo {
self.cache_max_age self.cache_max_age
} }
pub fn shard_dir(&self, publisher: &str) -> PathBuf {
self.root
.join("publisher")
.join(publisher)
.join("catalog2")
}
pub fn get_catalog_file_path(&self, publisher: &str, filename: &str) -> Result<PathBuf> { pub fn get_catalog_file_path(&self, publisher: &str, filename: &str) -> Result<PathBuf> {
let backend = self let backend = self
.backend .backend