CloudNebulaProject/ips
1
0
Fork 0
mirror of https://codeberg.org/Toasterson/ips.git synced 2026-04-10 13:20:42 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Refactor: Replace redb with rusqlite for catalog handling

Browse source 
- Transitioned the catalog backend from `redb` to `rusqlite` for better compatibility and concurrency.
- Updated `IpsProvider` to use SQLite for package querying, dependency resolution, and obsolescence handling.
- Removed `decode_manifest_bytes_local` and unused `manifest` cache logic.
- Simplified catalog-related functions and integrated `sqlite_catalog` module.
- Enhanced test functions and added schemas for managing SQLite databases.
This commit is contained in:
Till Wegmueller 2026-02-04 22:39:42 +01:00
parent 4ab529f4c7
commit def11a1dfb
No known key found for this signature in database
18 changed files with 1790 additions and 2106 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

114
Cargo.lock generated
View file

@ -326,26 +326,6 @@ version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bincode"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36eaf5d7b090263e8150820482d5d93cd964a81e4019913c972f4edcc6edb740"
dependencies = [
"bincode_derive",
"serde",
"unty",
]
[[package]]
name = "bincode_derive"
version = "2.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf95709a440f45e986983918d0e8a1f30a9b1df04918fc828670606804ac3c09"
dependencies = [
"virtue",
]
[[package]]
name = "bitflags"
version = "2.10.0"
@ -765,6 +745,18 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "fallible-iterator"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
[[package]]
name = "fastrand"
version = "2.3.0"
@ -1020,12 +1012,6 @@ dependencies = [
"tracing",
]
[[package]]
name = "half"
version = "1.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b43ede17f21864e81be2fa654110bf1e793774238d86ef8555c37e6519c0403"
[[package]]
name = "hashbrown"
version = "0.14.5"
@ -1051,6 +1037,15 @@ version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
[[package]]
name = "hashlink"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af"
dependencies = [
"hashbrown 0.14.5",
]
[[package]]
name = "heck"
version = "0.4.1"
@ -1463,7 +1458,7 @@ checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091"
name = "libips"
version = "0.5.3"
dependencies = [
"bincode",
"base64 0.22.1",
"chrono",
"diff-struct",
"flate2",
@ -1474,14 +1469,13 @@ dependencies = [
"object",
"pest",
"pest_derive",
"redb",
"regex",
"reqwest",
"resolvo",
"rusqlite",
"rust-ini",
"semver",
"serde",
"serde_cbor",
"serde_json",
"sha1",
"sha2",
@ -1503,6 +1497,17 @@ dependencies = [
"libc",
]
[[package]]
name = "libsqlite3-sys"
version = "0.28.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f"
dependencies = [
"cc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.11.0"
@ -2120,10 +2125,12 @@ dependencies = [
"opentelemetry_sdk",
"predicates",
"reqwest",
"rusqlite",
"rustls",
"serde",
"serde_json",
"sha1",
"sha2",
"socket2",
"tempfile",
"thiserror 2.0.17",
@ -2197,7 +2204,7 @@ dependencies = [
"reqwest",
"shellexpand",
"specfile",
"thiserror 1.0.69",
"thiserror 2.0.17",
"url",
"which",
]
@ -2411,15 +2418,6 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "redb"
version = "3.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae323eb086579a3769daa2c753bb96deb95993c534711e0dbe881b5192906a06"
dependencies = [
"libc",
]
[[package]]
name = "redox_syscall"
version = "0.5.18"
@ -2548,6 +2546,20 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "rusqlite"
version = "0.31.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b838eba278d213a8beaf485bd313fd580ca4505a00d5871caeb1457c55322cae"
dependencies = [
"bitflags",
"fallible-iterator",
"fallible-streaming-iterator",
"hashlink",
"libsqlite3-sys",
"smallvec",
]
[[package]]
name = "rust-ini"
version = "0.21.3"
@ -2729,16 +2741,6 @@ dependencies = [
"serde_derive",
]
[[package]]
name = "serde_cbor"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
dependencies = [
"half",
"serde",
]
[[package]]
name = "serde_core"
version = "1.0.228"
@ -2895,7 +2897,7 @@ dependencies = [
"anyhow",
"pest",
"pest_derive",
"thiserror 1.0.69",
"thiserror 2.0.17",
]
[[package]]
@ -3479,12 +3481,6 @@ version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
[[package]]
name = "unty"
version = "0.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d49784317cd0d1ee7ec5c716dd598ec5b4483ea832a2dced265471cc0f690ae"
[[package]]
name = "url"
version = "2.5.7"
@ -3554,12 +3550,6 @@ version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "virtue"
version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
[[package]]
name = "wait-timeout"
version = "0.2.1"

9
libips/Cargo.toml
View file

@ -32,20 +32,19 @@ pest_derive = "2.1.0"
strum = { version = "0.27", features = ["derive"] }
serde = { version = "1.0.207", features = ["derive"] }
serde_json = "1.0.124"
serde_cbor = "0.11.2"
flate2 = "1.0.28"
lz4 = "1.24.0"
base64 = "0.22"
semver = { version = "1.0.20", features = ["serde"] }
diff-struct = "0.5.3"
chrono = "0.4.41"
tempfile = "3.20.0"
walkdir = "2.4.0"
redb = { version = "3" }
bincode = { version = "2", features = ["serde"] }
rusqlite = { version = "0.31", default-features = false }
rust-ini = "0.21"
reqwest = { version = "0.12", features = ["blocking", "json", "gzip", "deflate"] }
resolvo = "0.10"
[features]
default = ["redb-index"]
redb-index = [] # Enable redb-based index for obsoleted packages
default = ["bundled-sqlite"]
bundled-sqlite = ["rusqlite/bundled"]

1165
libips/src/image/catalog.rs
View file

File diff suppressed because it is too large Load diff

338
libips/src/image/installed.rs
View file

@ -1,7 +1,8 @@
use crate::actions::Manifest;
use crate::fmri::Fmri;
use crate::repository::sqlite_catalog::INSTALLED_SCHEMA;
use miette::Diagnostic;
use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition};
use rusqlite::{Connection, OpenFlags};
use serde::{Deserialize, Serialize};
use std::fs;
use std::path::{Path, PathBuf};
@ -9,11 +10,6 @@ use std::str::FromStr;
use thiserror::Error;
use tracing::info;
/// Table definition for the installed packages database
/// Key: full FMRI including publisher (pkg://publisher/stem@version)
/// Value: serialized Manifest
pub const INSTALLED_TABLE: TableDefinition<&str, &[u8]> = TableDefinition::new("installed");
/// Errors that can occur when working with the installed packages database
#[derive(Error, Debug, Diagnostic)]
pub enum InstalledError {
@ -38,6 +34,12 @@ pub enum InstalledError {
PackageNotFound(String),
}
impl From<rusqlite::Error> for InstalledError {
fn from(e: rusqlite::Error) -> Self {
InstalledError::Database(format!("SQLite error: {}", e))
}
}
/// Result type for installed packages operations
pub type Result<T> = std::result::Result<T, InstalledError>;
@ -58,17 +60,6 @@ pub struct InstalledPackages {
}
impl InstalledPackages {
// Note on borrowing and redb:
// When using redb, there's a potential borrowing issue when working with transactions and tables.
// The issue occurs because:
// 1. Tables borrow from the transaction they were opened from
// 2. When committing a transaction with tx.commit(), the transaction is moved
// 3. If a table is still borrowing from the transaction when commit() is called, Rust's borrow checker will prevent the move
//
// To fix this issue, we use block scopes {} around table operations to ensure that the table
// objects are dropped (and their borrows released) before committing the transaction.
// This pattern is used in all methods that commit transactions after table operations.
/// Create a new installed packages database
pub fn new<P: AsRef<Path>>(db_path: P) -> Self {
InstalledPackages {
@ -78,50 +69,26 @@ impl InstalledPackages {
/// Dump the contents of the installed table to stdout for debugging
pub fn dump_installed_table(&self) -> Result<()> {
// Open the database
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
// Begin a read transaction
let tx = db
.begin_read()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
let mut stmt = conn.prepare("SELECT fmri, manifest FROM installed")?;
let mut rows = stmt.query([])?;
// Open the installed table
match tx.open_table(INSTALLED_TABLE) {
Ok(table) => {
let mut count = 0;
for entry_result in table.iter().map_err(|e| {
InstalledError::Database(format!("Failed to iterate installed table: {}", e))
})? {
let (key, value) = entry_result.map_err(|e| {
InstalledError::Database(format!(
"Failed to get entry from installed table: {}",
e
))
})?;
let key_str = key.value();
while let Some(row) = rows.next()? {
let fmri_str: String = row.get(0)?;
let manifest_bytes: Vec<u8> = row.get(1)?;
// Try to deserialize the manifest
match serde_json::from_slice::<Manifest>(value.value()) {
match serde_json::from_slice::<Manifest>(&manifest_bytes) {
Ok(manifest) => {
// Extract the publisher from the FMRI attribute
let publisher = manifest
.attributes
.iter()
.find(|attr| attr.key == "pkg.fmri")
.and_then(|attr| attr.values.first().cloned())
.unwrap_or_else(|| "unknown".to_string());
println!("Key: {}", key_str);
println!(" FMRI: {}", publisher);
println!("FMRI: {}", fmri_str);
println!(" Attributes: {}", manifest.attributes.len());
println!(" Files: {}", manifest.files.len());
println!(" Directories: {}", manifest.directories.len());
println!(" Dependencies: {}", manifest.dependencies.len());
}
Err(e) => {
println!("Key: {}", key_str);
println!("FMRI: {}", fmri_str);
println!(" Error deserializing manifest: {}", e);
}
}
@ -130,46 +97,13 @@ impl InstalledPackages {
println!("Total entries in installed table: {}", count);
Ok(())
}
Err(e) => {
println!("Error opening installed table: {}", e);
Err(InstalledError::Database(format!(
"Failed to open installed table: {}",
e
)))
}
}
}
/// Get database statistics
pub fn get_db_stats(&self) -> Result<()> {
// Open the database
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
// Begin a read transaction
let tx = db
.begin_read()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
let installed_count: i64 = conn.query_row("SELECT COUNT(*) FROM installed", [], |row| row.get(0))?;
// Get table statistics
let mut installed_count = 0;
// Count installed entries
if let Ok(table) = tx.open_table(INSTALLED_TABLE) {
for result in table.iter().map_err(|e| {
InstalledError::Database(format!("Failed to iterate installed table: {}", e))
})? {
let _ = result.map_err(|e| {
InstalledError::Database(format!(
"Failed to get entry from installed table: {}",
e
))
})?;
installed_count += 1;
}
}
// Print statistics
println!("Database path: {}", self.db_path.display());
println!("Table statistics:");
println!(" Installed table: {} entries", installed_count);
@ -180,72 +114,33 @@ impl InstalledPackages {
/// Initialize the installed packages database
pub fn init_db(&self) -> Result<()> {
// Create a parent directory if it doesn't exist
// Create parent directory if it doesn't exist
if let Some(parent) = self.db_path.parent() {
fs::create_dir_all(parent)?;
}
// Open or create the database
let db = Database::create(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to create database: {}", e)))?;
// Create or open the database
let conn = Connection::open(&self.db_path)?;
// Create tables
let tx = db
.begin_write()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
tx.open_table(INSTALLED_TABLE).map_err(|e| {
InstalledError::Database(format!("Failed to create installed table: {}", e))
})?;
tx.commit().map_err(|e| {
InstalledError::Database(format!("Failed to commit transaction: {}", e))
})?;
// Execute schema
conn.execute_batch(INSTALLED_SCHEMA)?;
Ok(())
}
/// Add a package to the installed packages database
pub fn add_package(&self, fmri: &Fmri, manifest: &Manifest) -> Result<()> {
// Open the database
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
let mut conn = Connection::open(&self.db_path)?;
// Begin a writing transaction
let tx = db
.begin_write()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
// Create the key (full FMRI including publisher)
let key = fmri.to_string();
// Serialize the manifest
let manifest_bytes = serde_json::to_vec(manifest)?;
// Use a block scope to ensure the table is dropped before committing the transaction
{
// Open the installed table
let mut installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| {
InstalledError::Database(format!("Failed to open installed table: {}", e))
})?;
// Insert the package into the installed table
installed_table
.insert(key.as_str(), manifest_bytes.as_slice())
.map_err(|e| {
InstalledError::Database(format!(
"Failed to insert into installed table: {}",
e
))
})?;
// The table is dropped at the end of this block, releasing its borrow of tx
}
// Commit the transaction
tx.commit().map_err(|e| {
InstalledError::Database(format!("Failed to commit transaction: {}", e))
})?;
let tx = conn.transaction()?;
tx.execute(
"INSERT OR REPLACE INTO installed (fmri, manifest) VALUES (?1, ?2)",
rusqlite::params![key, manifest_bytes],
)?;
tx.commit()?;
info!("Added package to installed database: {}", key);
Ok(())
@ -253,42 +148,24 @@ impl InstalledPackages {
/// Remove a package from the installed packages database
pub fn remove_package(&self, fmri: &Fmri) -> Result<()> {
// Open the database
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
let mut conn = Connection::open(&self.db_path)?;
// Begin a writing transaction
let tx = db
.begin_write()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
// Create the key (full FMRI including publisher)
let key = fmri.to_string();
// Use a block scope to ensure the table is dropped before committing the transaction
{
// Open the installed table
let mut installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| {
InstalledError::Database(format!("Failed to open installed table: {}", e))
})?;
// Check if the package exists
if let Ok(None) = installed_table.get(key.as_str()) {
let exists: bool = conn.query_row(
"SELECT EXISTS(SELECT 1 FROM installed WHERE fmri = ?1)",
rusqlite::params![key],
|row| row.get(0),
)?;
if !exists {
return Err(InstalledError::PackageNotFound(key));
}
// Remove the package from the installed table
installed_table.remove(key.as_str()).map_err(|e| {
InstalledError::Database(format!("Failed to remove from installed table: {}", e))
})?;
// The table is dropped at the end of this block, releasing its borrow of tx
}
// Commit the transaction
tx.commit().map_err(|e| {
InstalledError::Database(format!("Failed to commit transaction: {}", e))
})?;
let tx = conn.transaction()?;
tx.execute("DELETE FROM installed WHERE fmri = ?1", rusqlite::params![key])?;
tx.commit()?;
info!("Removed package from installed database: {}", key);
Ok(())
@ -296,127 +173,60 @@ impl InstalledPackages {
/// Query the installed packages database for packages matching a pattern
pub fn query_packages(&self, pattern: Option<&str>) -> Result<Vec<InstalledPackageInfo>> {
// Open the database
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
// Begin a read transaction
let tx = db
.begin_read()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
let query = if let Some(pattern) = pattern {
format!("SELECT fmri FROM installed WHERE fmri LIKE '%{}%'", pattern.replace('\'', "''"))
} else {
"SELECT fmri FROM installed".to_string()
};
// Use a block scope to ensure the table is dropped when no longer needed
let results = {
// Open the installed table
let installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| {
InstalledError::Database(format!("Failed to open installed table: {}", e))
})?;
let mut stmt = conn.prepare(&query)?;
let mut rows = stmt.query([])?;
let mut results = Vec::new();
// Process the installed table
// Iterate through all entries in the table
for entry_result in installed_table.iter().map_err(|e| {
InstalledError::Database(format!("Failed to iterate installed table: {}", e))
})? {
let (key, _) = entry_result.map_err(|e| {
InstalledError::Database(format!(
"Failed to get entry from installed table: {}",
e
))
})?;
let key_str = key.value();
// Skip if the key doesn't match the pattern
if let Some(pattern) = pattern {
if !key_str.contains(pattern) {
continue;
}
}
// Parse the key to get the FMRI
let fmri = Fmri::from_str(key_str)?;
// Get the publisher (handling the Option<String>)
let publisher = fmri
.publisher
.clone()
.unwrap_or_else(|| "unknown".to_string());
// Add to results
while let Some(row) = rows.next()? {
let fmri_str: String = row.get(0)?;
let fmri = Fmri::from_str(&fmri_str)?;
let publisher = fmri.publisher.clone().unwrap_or_else(|| "unknown".to_string());
results.push(InstalledPackageInfo { fmri, publisher });
}
results
// The table is dropped at the end of this block
};
Ok(results)
}
/// Get a manifest from the installed packages database
pub fn get_manifest(&self, fmri: &Fmri) -> Result<Option<Manifest>> {
// Open the database
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
// Begin a read transaction
let tx = db
.begin_read()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
// Create the key (full FMRI including publisher)
let key = fmri.to_string();
let result = conn.query_row(
"SELECT manifest FROM installed WHERE fmri = ?1",
rusqlite::params![key],
|row| {
let bytes: Vec<u8> = row.get(0)?;
Ok(bytes)
},
);
// Use a block scope to ensure the table is dropped when no longer needed
let manifest_option = {
// Open the installed table
let installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| {
InstalledError::Database(format!("Failed to open installed table: {}", e))
})?;
// Try to get the manifest from the installed table
if let Ok(Some(bytes)) = installed_table.get(key.as_str()) {
Some(serde_json::from_slice(bytes.value())?)
} else {
None
match result {
Ok(bytes) => Ok(Some(serde_json::from_slice(&bytes)?)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(e.into()),
}
// The table is dropped at the end of this block
};
Ok(manifest_option)
}
/// Check if a package is installed
pub fn is_installed(&self, fmri: &Fmri) -> Result<bool> {
// Open the database
let db = Database::open(&self.db_path)
.map_err(|e| InstalledError::Database(format!("Failed to open database: {}", e)))?;
let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
// Begin a read transaction
let tx = db
.begin_read()
.map_err(|e| InstalledError::Database(format!("Failed to begin transaction: {}", e)))?;
// Create the key (full FMRI including publisher)
let key = fmri.to_string();
let exists: bool = conn.query_row(
"SELECT EXISTS(SELECT 1 FROM installed WHERE fmri = ?1)",
rusqlite::params![key],
|row| row.get(0),
)?;
// Use a block scope to ensure the table is dropped when no longer needed
let is_installed = {
// Open the installed table
let installed_table = tx.open_table(INSTALLED_TABLE).map_err(|e| {
InstalledError::Database(format!("Failed to open installed table: {}", e))
})?;
// Check if the package exists
if let Ok(Some(_)) = installed_table.get(key.as_str()) {
true
} else {
false
}
// The table is dropped at the end of this block
};
Ok(is_installed)
Ok(exists)
}
}

18
libips/src/image/installed_tests.rs
View file

@ -1,7 +1,7 @@
use super::*;
use crate::actions::{Attr, Manifest};
use crate::fmri::Fmri;
use redb::{Database, ReadableTable};
use rusqlite::Connection;
use std::str::FromStr;
use tempfile::tempdir;
@ -82,7 +82,7 @@ fn test_installed_packages() {
fn test_installed_packages_key_format() {
// Create a temporary directory for the test
let temp_dir = tempdir().unwrap();
let db_path = temp_dir.path().join("installed.redb");
let db_path = temp_dir.path().join("installed.db");
// Create the installed packages database
let installed = InstalledPackages::new(&db_path);
@ -104,15 +104,15 @@ fn test_installed_packages_key_format() {
installed.add_package(&fmri, &manifest).unwrap();
// Open the database directly to check the key format
let db = Database::open(&db_path).unwrap();
let tx = db.begin_read().unwrap();
let table = tx.open_table(installed::INSTALLED_TABLE).unwrap();
let conn = Connection::open(&db_path).unwrap();
let mut stmt = conn.prepare("SELECT fmri FROM installed").unwrap();
let mut rows = stmt.query([]).unwrap();
// Iterate through the keys
// Collect the keys
let mut keys = Vec::new();
for entry in table.iter().unwrap() {
let (key, _) = entry.unwrap();
keys.push(key.value().to_string());
while let Some(row) = rows.next().unwrap() {
let fmri_str: String = row.get(0).unwrap();
keys.push(fmri_str);
}
// Verify that there is one key and it has the correct format

166
libips/src/image/mod.rs
View file

@ -4,7 +4,7 @@ mod tests;
use miette::Diagnostic;
use properties::*;
use redb::{Database, ReadableDatabase, ReadableTable};
use rusqlite::{Connection, OpenFlags};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::fs::{self, File};
@ -15,7 +15,7 @@ use crate::repository::{FileBackend, ReadableRepository, RepositoryError, RestBa
// Export the catalog module
pub mod catalog;
use catalog::{INCORPORATE_TABLE, ImageCatalog, PackageInfo};
use catalog::{ImageCatalog, PackageInfo};
// Export the installed packages module
pub mod installed;
@ -79,6 +79,12 @@ pub enum ImageError {
NoPublishers,
}
impl From<rusqlite::Error> for ImageError {
fn from(e: rusqlite::Error) -> Self {
ImageError::Database(format!("SQLite error: {}", e))
}
}
pub type Result<T> = std::result::Result<T, ImageError>;
/// Type of image, either Full (base path of "/") or Partial (attached to a full image)
@ -281,7 +287,7 @@ impl Image {
/// Returns the path to the installed packages database
pub fn installed_db_path(&self) -> PathBuf {
self.metadata_dir().join("installed.redb")
self.metadata_dir().join("installed.db")
}
/// Returns the path to the manifest directory
@ -294,14 +300,31 @@ impl Image {
self.metadata_dir().join("catalog")
}
/// Returns the path to the catalog database
pub fn catalog_db_path(&self) -> PathBuf {
self.metadata_dir().join("catalog.redb")
/// Returns the path to the active catalog database (packages and dependencies)
pub fn active_db_path(&self) -> PathBuf {
self.metadata_dir().join("active.db")
}
/// Returns the path to the obsoleted packages database (separate DB)
/// Returns the path to the obsoleted packages database
pub fn obsolete_db_path(&self) -> PathBuf {
self.metadata_dir().join("obsolete.db")
}
/// Returns the path to the full-text search database
pub fn fts_db_path(&self) -> PathBuf {
self.metadata_dir().join("fts.db")
}
/// Deprecated: Use active_db_path() instead
#[deprecated(note = "Use active_db_path() instead")]
pub fn catalog_db_path(&self) -> PathBuf {
self.active_db_path()
}
/// Deprecated: Use obsolete_db_path() instead
#[deprecated(note = "Use obsolete_db_path() instead")]
pub fn obsoleted_db_path(&self) -> PathBuf {
self.metadata_dir().join("obsoleted.redb")
self.obsolete_db_path()
}
/// Creates the metadata directory if it doesn't exist
@ -528,14 +551,18 @@ impl Image {
/// Initialize the catalog database
pub fn init_catalog_db(&self) -> Result<()> {
let catalog = ImageCatalog::new(
self.catalog_dir(),
self.catalog_db_path(),
self.obsoleted_db_path(),
);
catalog.init_db().map_err(|e| {
ImageError::Database(format!("Failed to initialize catalog database: {}", e))
})
use crate::repository::sqlite_catalog::ACTIVE_SCHEMA;
let path = self.active_db_path();
if let Some(parent) = path.parent() {
fs::create_dir_all(parent)?;
}
let conn = Connection::open(&path)?;
conn.execute_batch(ACTIVE_SCHEMA)
.map_err(|e| ImageError::Database(format!("Failed to initialize catalog database: {}", e)))?;
Ok(())
}
/// Download catalogs from all configured publishers and build the merged catalog
@ -657,65 +684,102 @@ impl Image {
/// Look up an incorporation lock for a given stem.
/// Returns Some(release) if a lock exists, otherwise None.
pub fn get_incorporated_release(&self, stem: &str) -> Result<Option<String>> {
let db = Database::open(self.catalog_db_path())
let conn = Connection::open_with_flags(
&self.active_db_path(),
OpenFlags::SQLITE_OPEN_READ_ONLY,
)
.map_err(|e| ImageError::Database(format!("Failed to open catalog database: {}", e)))?;
let tx = db.begin_read().map_err(|e| {
ImageError::Database(format!("Failed to begin read transaction: {}", e))
})?;
match tx.open_table(INCORPORATE_TABLE) {
Ok(table) => match table.get(stem) {
Ok(Some(val)) => Ok(Some(String::from_utf8_lossy(val.value()).to_string())),
Ok(None) => Ok(None),
let result = conn.query_row(
"SELECT release FROM incorporate_locks WHERE stem = ?1",
rusqlite::params![stem],
|row| row.get(0),
);
match result {
Ok(release) => Ok(Some(release)),
Err(rusqlite::Error::QueryReturnedNoRows) => Ok(None),
Err(e) => Err(ImageError::Database(format!(
"Failed to read incorporate lock: {}",
e
))),
},
Err(_) => Ok(None),
}
}
/// Add an incorporation lock for a stem to a specific release.
/// Fails if a lock already exists for the stem.
/// Uses INSERT OR REPLACE, so will update if a lock already exists.
pub fn add_incorporation_lock(&self, stem: &str, release: &str) -> Result<()> {
let db = Database::open(self.catalog_db_path())
let mut conn = Connection::open(&self.active_db_path())
.map_err(|e| ImageError::Database(format!("Failed to open catalog database: {}", e)))?;
let tx = db.begin_write().map_err(|e| {
let tx = conn.transaction().map_err(|e| {
ImageError::Database(format!("Failed to begin write transaction: {}", e))
})?;
{
let mut table = tx.open_table(INCORPORATE_TABLE).map_err(|e| {
ImageError::Database(format!("Failed to open incorporate table: {}", e))
})?;
if let Ok(Some(_)) = table.get(stem) {
return Err(ImageError::Database(format!(
"Incorporation lock already exists for stem {}",
stem
)));
}
table.insert(stem, release.as_bytes()).map_err(|e| {
ImageError::Database(format!("Failed to insert incorporate lock: {}", e))
})?;
}
tx.execute(
"INSERT OR REPLACE INTO incorporate_locks (stem, release) VALUES (?1, ?2)",
rusqlite::params![stem, release],
)
.map_err(|e| ImageError::Database(format!("Failed to insert incorporate lock: {}", e)))?;
tx.commit().map_err(|e| {
ImageError::Database(format!("Failed to commit incorporate lock: {}", e))
})?;
Ok(())
}
/// Get a manifest from the catalog
/// Get a manifest from the catalog.
/// First checks the local manifest cache on disk, then falls back to repository fetch.
/// Note: active.db does NOT store manifest blobs - manifests are served from the repository.
pub fn get_manifest_from_catalog(
&self,
fmri: &crate::fmri::Fmri,
) -> Result<Option<crate::actions::Manifest>> {
let catalog = ImageCatalog::new(
self.catalog_dir(),
self.catalog_db_path(),
self.obsoleted_db_path(),
);
catalog.get_manifest(fmri).map_err(|e| {
ImageError::Database(format!("Failed to get manifest from catalog: {}", e))
// Helper to URL-encode filename components
fn url_encode(s: &str) -> String {
s.chars()
.map(|c| match c {
'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(),
' ' => "+".to_string(),
_ => {
let mut buf = [0u8; 4];
let bytes = c.encode_utf8(&mut buf).as_bytes();
bytes.iter().map(|b| format!("%{:02X}", b)).collect()
}
})
.collect()
}
// Check local manifest cache on disk
let publisher = fmri.publisher.as_deref().unwrap_or("");
let manifest_dir = self.manifest_dir().join(publisher);
let stem_encoded = url_encode(fmri.stem());
let version_encoded = url_encode(&fmri.version());
let manifest_path = manifest_dir.join(format!("{}@{}.p5m", stem_encoded, version_encoded));
if manifest_path.exists() {
let content = fs::read_to_string(&manifest_path)?;
let manifest = crate::actions::Manifest::parse_string(content)
.map_err(|e| ImageError::Database(format!("Failed to parse manifest: {}", e)))?;
return Ok(Some(manifest));
}
// Check catalog shards for a minimal manifest
let catalog = crate::image::catalog::ImageCatalog::new(
self.catalog_dir(),
self.active_db_path(),
self.obsolete_db_path(),
);
if let Ok(Some(manifest)) = catalog.get_manifest(fmri) {
return Ok(Some(manifest));
}
// Fall back to repository fetch
match self.get_manifest_from_repository(fmri) {
Ok(manifest) => Ok(Some(manifest)),
Err(_) => Ok(None),
}
}
/// Fetch a full manifest for the given FMRI directly from its repository origin.

43
libips/src/repository/file_backend.rs
View file

@ -1899,11 +1899,25 @@ impl WritableRepository for FileBackend {
if !no_catalog {
info!("Rebuilding catalog...");
self.rebuild_catalog(&pub_name, true)?;
// Build SQLite catalog shards (active.db, obsolete.db, fts.db)
info!("Building catalog shards...");
let catalog_dir = Self::construct_catalog_path(&self.path, &pub_name);
let shard_dir = self.shard_dir(&pub_name);
crate::repository::sqlite_catalog::build_shards(
&catalog_dir,
&pub_name,
&shard_dir,
)
.map_err(|e| {
RepositoryError::Other(format!("Failed to build catalog shards: {}", e.message))
})?;
}
if !no_index {
info!("Rebuilding search index...");
self.build_search_index(&pub_name)?;
// FTS index is now built as part of catalog shards (fts.db)
// No separate index building needed
info!("Search index built as part of catalog shards (fts.db)");
}
}
@ -1929,11 +1943,25 @@ impl WritableRepository for FileBackend {
if !no_catalog {
info!("Refreshing catalog...");
self.rebuild_catalog(&pub_name, true)?;
// Build SQLite catalog shards (active.db, obsolete.db, fts.db)
info!("Building catalog shards...");
let catalog_dir = Self::construct_catalog_path(&self.path, &pub_name);
let shard_dir = self.shard_dir(&pub_name);
crate::repository::sqlite_catalog::build_shards(
&catalog_dir,
&pub_name,
&shard_dir,
)
.map_err(|e| {
RepositoryError::Other(format!("Failed to build catalog shards: {}", e.message))
})?;
}
if !no_index {
info!("Refreshing search index...");
self.build_search_index(&pub_name)?;
// FTS index is now built as part of catalog shards (fts.db)
// No separate index building needed
info!("Search index built as part of catalog shards (fts.db)");
}
}
@ -2166,6 +2194,13 @@ impl FileBackend {
base_path.join("publisher").join(publisher).join("catalog")
}
/// Helper method to construct a shard directory path for catalog v2 shards
///
/// Format: base_path/publisher/publisher_name/catalog2
pub fn shard_dir(&self, publisher: &str) -> PathBuf {
self.path.join("publisher").join(publisher).join("catalog2")
}
/// Helper method to construct a manifest path consistently
///
/// Format: base_path/publisher/publisher_name/pkg/stem/encoded_version

49
libips/src/repository/mod.rs
View file

@ -168,60 +168,21 @@ impl From<StripPrefixError> for RepositoryError {
}
}
// Implement From for redb error types
impl From<redb::Error> for RepositoryError {
fn from(err: redb::Error) -> Self {
// Implement From for rusqlite error types
impl From<rusqlite::Error> for RepositoryError {
fn from(err: rusqlite::Error) -> Self {
RepositoryError::Other(format!("Database error: {}", err))
}
}
impl From<redb::DatabaseError> for RepositoryError {
fn from(err: redb::DatabaseError) -> Self {
RepositoryError::Other(format!("Database error: {}", err))
}
}
impl From<redb::TransactionError> for RepositoryError {
fn from(err: redb::TransactionError) -> Self {
RepositoryError::Other(format!("Transaction error: {}", err))
}
}
impl From<redb::TableError> for RepositoryError {
fn from(err: redb::TableError) -> Self {
RepositoryError::Other(format!("Table error: {}", err))
}
}
impl From<redb::StorageError> for RepositoryError {
fn from(err: redb::StorageError) -> Self {
RepositoryError::Other(format!("Storage error: {}", err))
}
}
impl From<redb::CommitError> for RepositoryError {
fn from(err: redb::CommitError) -> Self {
RepositoryError::Other(format!("Commit error: {}", err))
}
}
impl From<bincode::error::DecodeError> for RepositoryError {
fn from(err: bincode::error::DecodeError) -> Self {
RepositoryError::Other(format!("Serialization error: {}", err))
}
}
impl From<bincode::error::EncodeError> for RepositoryError {
fn from(err: bincode::error::EncodeError) -> Self {
RepositoryError::Other(format!("Serialization error: {}", err))
}
}
pub mod catalog;
mod catalog_writer;
pub(crate) mod file_backend;
mod obsoleted;
pub mod progress;
mod rest_backend;
pub mod shard_sync;
pub mod sqlite_catalog;
#[cfg(test)]
mod tests;

544
libips/src/repository/obsoleted.rs
View file

@ -1,11 +1,11 @@
use crate::fmri::Fmri;
use crate::repository::sqlite_catalog::OBSOLETED_INDEX_SCHEMA;
use crate::repository::{RepositoryError, Result};
use chrono::{DateTime, Duration as ChronoDuration, Utc};
use miette::Diagnostic;
use redb::{Database, ReadableDatabase, ReadableTable, TableDefinition};
use regex::Regex;
use rusqlite::{Connection, OpenFlags};
use serde::{Deserialize, Serialize};
use serde_cbor;
use serde_json;
use sha2::Digest;
use std::fs;
@ -176,54 +176,12 @@ impl From<crate::fmri::FmriError> for ObsoletedPackageError {
}
}
impl From<redb::Error> for ObsoletedPackageError {
fn from(err: redb::Error) -> Self {
impl From<rusqlite::Error> for ObsoletedPackageError {
fn from(err: rusqlite::Error) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string())
}
}
impl From<redb::DatabaseError> for ObsoletedPackageError {
fn from(err: redb::DatabaseError) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string())
}
}
impl From<redb::TransactionError> for ObsoletedPackageError {
fn from(err: redb::TransactionError) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string())
}
}
impl From<redb::TableError> for ObsoletedPackageError {
fn from(err: redb::TableError) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string())
}
}
impl From<redb::StorageError> for ObsoletedPackageError {
fn from(err: redb::StorageError) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string())
}
}
impl From<redb::CommitError> for ObsoletedPackageError {
fn from(err: redb::CommitError) -> Self {
ObsoletedPackageError::DatabaseError(err.to_string())
}
}
impl From<bincode::error::EncodeError> for ObsoletedPackageError {
fn from(err: bincode::error::EncodeError) -> Self {
ObsoletedPackageError::SerializationError(err.to_string())
}
}
impl From<bincode::error::DecodeError> for ObsoletedPackageError {
fn from(err: bincode::error::DecodeError) -> Self {
ObsoletedPackageError::SerializationError(err.to_string())
}
}
// Implement From<ObsoletedPackageError> for RepositoryError to allow conversion
// This makes it easier to use ObsoletedPackageError with the existing Result type
impl From<ObsoletedPackageError> for RepositoryError {
@ -325,19 +283,11 @@ impl ObsoletedPackageKey {
}
}
// Table definitions for the redb database
// Table for mapping FMRI directly to metadata
static FMRI_TO_METADATA_TABLE: TableDefinition<&[u8], &[u8]> =
TableDefinition::new("fmri_to_metadata");
// Table for mapping content hash to manifest (for non-NULL_HASH entries)
static HASH_TO_MANIFEST_TABLE: TableDefinition<&str, &str> =
TableDefinition::new("hash_to_manifest");
/// Index of obsoleted packages using redb for faster lookups and content-addressable storage
/// Index of obsoleted packages using SQLite for faster lookups and content-addressable storage
#[derive(Debug)]
struct RedbObsoletedPackageIndex {
/// The redb database
db: Database,
struct SqliteObsoletedPackageIndex {
/// Path to the SQLite database file
db_path: PathBuf,
/// Last time the index was accessed
last_accessed: Instant,
/// Whether the index is dirty and needs to be rebuilt
@ -346,26 +296,18 @@ struct RedbObsoletedPackageIndex {
max_age: Duration,
}
impl RedbObsoletedPackageIndex {
/// Create a new RedbObsoletedPackageIndex
impl SqliteObsoletedPackageIndex {
/// Create a new SqliteObsoletedPackageIndex
fn new<P: AsRef<Path>>(base_path: P) -> Result<Self> {
let db_path = base_path.as_ref().join("index.redb");
debug!("Creating redb database at {}", db_path.display());
let db_path = base_path.as_ref().join("index.db");
debug!("Creating SQLite database at {}", db_path.display());
// Create the database
let db = Database::create(&db_path)?;
// Create the tables if they don't exist
let write_txn = db.begin_write()?;
{
// Create the new table for direct FMRI to metadata mapping
write_txn.open_table(FMRI_TO_METADATA_TABLE)?;
write_txn.open_table(HASH_TO_MANIFEST_TABLE)?;
}
write_txn.commit()?;
// Create the database and tables
let conn = Connection::open(&db_path)?;
conn.execute_batch(OBSOLETED_INDEX_SCHEMA)?;
Ok(Self {
db,
db_path,
last_accessed: Instant::now(),
dirty: false,
max_age: Duration::from_secs(300), // 5 minutes
@ -377,12 +319,12 @@ impl RedbObsoletedPackageIndex {
self.dirty || self.last_accessed.elapsed() > self.max_age
}
/// Create an empty temporary file-based RedbObsoletedPackageIndex
/// Create an empty temporary file-based SqliteObsoletedPackageIndex
///
/// This is used as a fallback when the database creation fails.
/// It creates a database in a temporary directory that can be used temporarily.
fn empty() -> Self {
debug!("Creating empty temporary file-based redb database");
debug!("Creating empty temporary file-based SQLite database");
// Create a temporary directory
let temp_dir = tempfile::tempdir().unwrap_or_else(|e| {
@ -391,50 +333,44 @@ impl RedbObsoletedPackageIndex {
});
// Create a database file in the temporary directory
let db_path = temp_dir.path().join("empty.redb");
let db_path = temp_dir.path().join("empty.db");
// Create the database
let db = Database::create(&db_path).unwrap_or_else(|e| {
// Create the database and tables
let conn = Connection::open(&db_path).unwrap_or_else(|e| {
error!("Failed to create temporary database: {}", e);
panic!("Failed to create temporary database: {}", e);
});
// Create the tables
let write_txn = db.begin_write().unwrap();
{
// Create the new table for direct FMRI to metadata mapping
let _ = write_txn.open_table(FMRI_TO_METADATA_TABLE).unwrap();
let _ = write_txn.open_table(HASH_TO_MANIFEST_TABLE).unwrap();
}
write_txn.commit().unwrap();
conn.execute_batch(OBSOLETED_INDEX_SCHEMA).unwrap();
Self {
db,
db_path,
last_accessed: Instant::now(),
dirty: false,
max_age: Duration::from_secs(300), // 5 minutes
}
}
/// Open an existing RedbObsoletedPackageIndex
/// Open an existing SqliteObsoletedPackageIndex
fn open<P: AsRef<Path>>(base_path: P) -> Result<Self> {
let db_path = base_path.as_ref().join("index.redb");
debug!("Opening redb database at {}", db_path.display());
let db_path = base_path.as_ref().join("index.db");
debug!("Opening SQLite database at {}", db_path.display());
// Open the database
let db = Database::open(&db_path)?;
// Open the database (creating tables if they don't exist)
let conn = Connection::open(&db_path)?;
conn.execute_batch(OBSOLETED_INDEX_SCHEMA)?;
Ok(Self {
db,
db_path,
last_accessed: Instant::now(),
dirty: false,
max_age: Duration::from_secs(300), // 5 minutes
})
}
/// Create or open a RedbObsoletedPackageIndex
/// Create or open a SqliteObsoletedPackageIndex
fn create_or_open<P: AsRef<Path>>(base_path: P) -> Result<Self> {
let db_path = base_path.as_ref().join("index.redb");
let db_path = base_path.as_ref().join("index.db");
if db_path.exists() {
Self::open(base_path)
@ -464,69 +400,46 @@ impl RedbObsoletedPackageIndex {
metadata.content_hash.clone()
};
// Use the FMRI string directly as the key
let key_bytes = metadata.fmri.as_bytes();
// Serialize obsoleted_by as JSON string (or NULL if None)
let obsoleted_by_json = metadata
.obsoleted_by
.as_ref()
.map(|obs| serde_json::to_string(obs))
.transpose()?;
let metadata_bytes = match serde_cbor::to_vec(metadata) {
Ok(bytes) => bytes,
Err(e) => {
error!("Failed to serialize metadata with CBOR: {}", e);
return Err(ObsoletedPackageError::SerializationError(format!(
"Failed to serialize metadata with CBOR: {}",
e
))
.into());
}
};
let mut conn = Connection::open(&self.db_path)?;
let tx = conn.transaction()?;
// Begin write transaction
let write_txn = match self.db.begin_write() {
Ok(txn) => txn,
Err(e) => {
error!("Failed to begin write transaction: {}", e);
return Err(e.into());
}
};
{
// Open the tables
let mut fmri_to_metadata = match write_txn.open_table(FMRI_TO_METADATA_TABLE) {
Ok(table) => table,
Err(e) => {
error!("Failed to open FMRI_TO_METADATA_TABLE: {}", e);
return Err(e.into());
}
};
let mut hash_to_manifest = match write_txn.open_table(HASH_TO_MANIFEST_TABLE) {
Ok(table) => table,
Err(e) => {
error!("Failed to open HASH_TO_MANIFEST_TABLE: {}", e);
return Err(e.into());
}
};
// Insert the metadata directly with FMRI as the key
// This is the new approach that eliminates the intermediate hash lookup
if let Err(e) = fmri_to_metadata.insert(key_bytes, metadata_bytes.as_slice()) {
error!("Failed to insert into FMRI_TO_METADATA_TABLE: {}", e);
return Err(e.into());
}
// Insert into obsoleted_packages table
tx.execute(
"INSERT OR REPLACE INTO obsoleted_packages (
fmri, publisher, stem, version, status, obsolescence_date,
deprecation_message, obsoleted_by, metadata_version, content_hash
) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)",
rusqlite::params![
&metadata.fmri,
&key.publisher,
&key.stem,
&key.version,
&metadata.status,
&metadata.obsolescence_date,
metadata.deprecation_message.as_deref(),
obsoleted_by_json.as_deref(),
metadata.metadata_version,
&content_hash,
],
)?;
// Only store the manifest if it's not a NULL_HASH entry
// For NULL_HASH entries, a minimal manifest will be generated when requested
if content_hash != NULL_HASH {
if let Err(e) = hash_to_manifest.insert(content_hash.as_str(), manifest) {
error!("Failed to insert into HASH_TO_MANIFEST_TABLE: {}", e);
return Err(e.into());
}
}
tx.execute(
"INSERT OR REPLACE INTO obsoleted_manifests (content_hash, manifest) VALUES (?1, ?2)",
rusqlite::params![&content_hash, manifest],
)?;
}
if let Err(e) = write_txn.commit() {
error!("Failed to commit transaction: {}", e);
return Err(e.into());
}
tx.commit()?;
debug!("Successfully added entry to index: {}", metadata.fmri);
Ok(())
@ -536,32 +449,25 @@ impl RedbObsoletedPackageIndex {
fn remove_entry(&self, key: &ObsoletedPackageKey) -> Result<bool> {
// Use the FMRI string directly as the key
let fmri = key.to_fmri_string();
let key_bytes = fmri.as_bytes();
// First, check if the key exists in the new table
let exists_in_new_table = {
let read_txn = self.db.begin_read()?;
let fmri_to_metadata = read_txn.open_table(FMRI_TO_METADATA_TABLE)?;
let conn = Connection::open(&self.db_path)?;
fmri_to_metadata.get(key_bytes)?.is_some()
};
// Check if the entry exists
let exists: bool = conn.query_row(
"SELECT EXISTS(SELECT 1 FROM obsoleted_packages WHERE fmri = ?1)",
rusqlite::params![&fmri],
|row| row.get(0),
)?;
// If the key doesn't exist in either table, return early
if !exists_in_new_table {
if !exists {
return Ok(false);
}
// Now perform the actual removal
let write_txn = self.db.begin_write()?;
{
// Remove the entry from the new table
if exists_in_new_table {
let mut fmri_to_metadata = write_txn.open_table(FMRI_TO_METADATA_TABLE)?;
fmri_to_metadata.remove(key_bytes)?;
}
}
write_txn.commit()?;
// Remove the entry
conn.execute(
"DELETE FROM obsoleted_packages WHERE fmri = ?1",
rusqlite::params![&fmri],
)?;
Ok(true)
}
@ -573,43 +479,61 @@ impl RedbObsoletedPackageIndex {
) -> Result<Option<(ObsoletedPackageMetadata, String)>> {
// Use the FMRI string directly as the key
let fmri = key.to_fmri_string();
let key_bytes = fmri.as_bytes();
// First, try to get the metadata directly from the new table
let metadata_result = {
let read_txn = self.db.begin_read()?;
let fmri_to_metadata = read_txn.open_table(FMRI_TO_METADATA_TABLE)?;
let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
// Get the metadata bytes
match fmri_to_metadata.get(key_bytes)? {
Some(bytes) => {
// Convert to owned bytes before the transaction is dropped
let metadata_bytes = bytes.value().to_vec();
// Try to deserialize the metadata
match serde_cbor::from_slice::<ObsoletedPackageMetadata>(&metadata_bytes) {
Ok(metadata) => Some(metadata),
Err(e) => {
warn!(
"Failed to deserialize metadata from FMRI_TO_METADATA_TABLE with CBOR: {}",
e
);
None
}
}
}
None => None,
}
// Try to get the metadata from the database
let metadata_result = match conn.query_row(
"SELECT fmri, status, obsolescence_date, deprecation_message,
obsoleted_by, metadata_version, content_hash
FROM obsoleted_packages WHERE fmri = ?1",
rusqlite::params![&fmri],
|row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
row.get::<_, Option<String>>(3)?,
row.get::<_, Option<String>>(4)?,
row.get::<_, u32>(5)?,
row.get::<_, String>(6)?,
))
},
) {
Ok(result) => Some(result),
Err(rusqlite::Error::QueryReturnedNoRows) => None,
Err(e) => return Err(e.into()),
};
// If we found the metadata in the new table, use it
if let Some(metadata) = metadata_result {
// Get the content hash from the metadata
let content_hash = metadata.content_hash.clone();
if let Some((
fmri,
status,
obsolescence_date,
deprecation_message,
obsoleted_by_json,
metadata_version,
content_hash,
)) = metadata_result
{
// Deserialize obsoleted_by from JSON if present
let obsoleted_by = obsoleted_by_json
.as_ref()
.map(|json| serde_json::from_str::<Vec<String>>(json))
.transpose()?;
let metadata = ObsoletedPackageMetadata {
fmri,
status,
obsolescence_date,
deprecation_message,
obsoleted_by,
metadata_version,
content_hash: content_hash.clone(),
};
// For NULL_HASH entries, generate a minimal manifest
let manifest_str = if content_hash == NULL_HASH {
// Generate a minimal manifest for NULL_HASH entries
// Construct an FMRI string from the metadata
format!(
r#"{{
"attributes": [
@ -631,13 +555,13 @@ impl RedbObsoletedPackageIndex {
)
} else {
// For non-NULL_HASH entries, get the manifest from the database
let read_txn = self.db.begin_read()?;
let hash_to_manifest = read_txn.open_table(HASH_TO_MANIFEST_TABLE)?;
// Get the manifest string
match hash_to_manifest.get(content_hash.as_str())? {
Some(manifest) => manifest.value().to_string(),
None => {
match conn.query_row(
"SELECT manifest FROM obsoleted_manifests WHERE content_hash = ?1",
rusqlite::params![&content_hash],
|row| row.get::<_, String>(0),
) {
Ok(manifest) => manifest,
Err(rusqlite::Error::QueryReturnedNoRows) => {
warn!(
"Manifest not found for content hash: {}, generating minimal manifest",
content_hash
@ -663,6 +587,7 @@ impl RedbObsoletedPackageIndex {
metadata.fmri
)
}
Err(e) => return Err(e.into()),
}
};
Ok(Some((metadata, manifest_str)))
@ -676,33 +601,52 @@ impl RedbObsoletedPackageIndex {
&self,
) -> Result<Vec<(ObsoletedPackageKey, ObsoletedPackageMetadata, String)>> {
let mut entries = Vec::new();
let mut processed_keys = std::collections::HashSet::new();
// First, collect all entries from the new table
{
let read_txn = self.db.begin_read()?;
let fmri_to_metadata = read_txn.open_table(FMRI_TO_METADATA_TABLE)?;
let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
let mut iter = fmri_to_metadata.iter()?;
let mut stmt = conn.prepare(
"SELECT fmri, publisher, stem, version, status, obsolescence_date,
deprecation_message, obsoleted_by, metadata_version, content_hash
FROM obsoleted_packages",
)?;
while let Some(entry) = iter.next() {
let (key_bytes, metadata_bytes) = entry?;
let rows = stmt.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
row.get::<_, String>(3)?,
row.get::<_, String>(4)?,
row.get::<_, String>(5)?,
row.get::<_, Option<String>>(6)?,
row.get::<_, Option<String>>(7)?,
row.get::<_, u32>(8)?,
row.get::<_, String>(9)?,
))
})?;
// Convert to owned types before the transaction is dropped
let key_data = key_bytes.value().to_vec();
let metadata_data = metadata_bytes.value().to_vec();
// Convert key bytes to string and parse as FMRI
let fmri_str = match std::str::from_utf8(&key_data) {
Ok(s) => s,
for row_result in rows {
let (
fmri,
_publisher,
_stem,
_version,
status,
obsolescence_date,
deprecation_message,
obsoleted_by_json,
metadata_version,
content_hash,
) = match row_result {
Ok(r) => r,
Err(e) => {
warn!("Failed to convert key bytes to string: {}", e);
warn!("Failed to read row: {}", e);
continue;
}
};
// Parse the FMRI string to create an ObsoletedPackageKey
let key = match ObsoletedPackageKey::from_fmri_string(fmri_str) {
let key = match ObsoletedPackageKey::from_fmri_string(&fmri) {
Ok(key) => key,
Err(e) => {
warn!("Failed to parse FMRI string: {}", e);
@ -710,24 +654,28 @@ impl RedbObsoletedPackageIndex {
}
};
let metadata: ObsoletedPackageMetadata = match serde_cbor::from_slice(
&metadata_data,
) {
Ok(metadata) => metadata,
// Deserialize obsoleted_by from JSON if present
let obsoleted_by = match obsoleted_by_json
.as_ref()
.map(|json| serde_json::from_str::<Vec<String>>(json))
.transpose()
{
Ok(obs) => obs,
Err(e) => {
warn!(
"Failed to deserialize metadata from FMRI_TO_METADATA_TABLE with CBOR: {}",
e
);
warn!("Failed to deserialize obsoleted_by JSON: {}", e);
continue;
}
};
// Add the key to the set of processed keys
processed_keys.insert(key_data);
// Get the content hash from the metadata
let content_hash = metadata.content_hash.clone();
let metadata = ObsoletedPackageMetadata {
fmri: fmri.clone(),
status,
obsolescence_date,
deprecation_message,
obsoleted_by,
metadata_version,
content_hash: content_hash.clone(),
};
// For NULL_HASH entries, generate a minimal manifest
let manifest_str = if content_hash == NULL_HASH {
@ -749,16 +697,17 @@ impl RedbObsoletedPackageIndex {
}}
]
}}"#,
metadata.fmri
fmri
)
} else {
// For non-NULL_HASH entries, get the manifest from the database
let hash_to_manifest = read_txn.open_table(HASH_TO_MANIFEST_TABLE)?;
// Get the manifest string
match hash_to_manifest.get(content_hash.as_str())? {
Some(manifest) => manifest.value().to_string(),
None => {
match conn.query_row(
"SELECT manifest FROM obsoleted_manifests WHERE content_hash = ?1",
rusqlite::params![&content_hash],
|row| row.get::<_, String>(0),
) {
Ok(manifest) => manifest,
Err(rusqlite::Error::QueryReturnedNoRows) => {
warn!(
"Manifest not found for content hash: {}, generating minimal manifest",
content_hash
@ -781,15 +730,18 @@ impl RedbObsoletedPackageIndex {
}}
]
}}"#,
metadata.fmri
fmri
)
}
Err(e) => {
warn!("Failed to get manifest for content hash: {}", e);
continue;
}
}
};
entries.push((key, metadata, manifest_str));
}
}
Ok(entries)
}
@ -858,86 +810,26 @@ impl RedbObsoletedPackageIndex {
/// Clear the index
fn clear(&self) -> Result<()> {
// Begin a writing transaction
let write_txn = self.db.begin_write()?;
{
// Clear all tables by removing all entries
// Since redb doesn't have a clear() method, we need to iterate and remove each key
let conn = Connection::open(&self.db_path)?;
// Clear hash_to_manifest table
{
let mut hash_to_manifest = write_txn.open_table(FMRI_TO_METADATA_TABLE)?;
let keys_to_remove = {
// First, collect all keys in a separate scope
let read_txn = self.db.begin_read()?;
let hash_to_manifest_read = read_txn.open_table(FMRI_TO_METADATA_TABLE)?;
let mut keys = Vec::new();
let mut iter = hash_to_manifest_read.iter()?;
while let Some(entry) = iter.next() {
let (key, _) = entry?;
keys.push(key.value().to_vec());
}
keys
};
// Then remove all keys
for key in keys_to_remove {
hash_to_manifest.remove(key.as_slice())?;
}
}
// Clear hash_to_manifest table
{
let mut hash_to_manifest = write_txn.open_table(HASH_TO_MANIFEST_TABLE)?;
let keys_to_remove = {
// First, collect all keys in a separate scope
let read_txn = self.db.begin_read()?;
let hash_to_manifest_read = read_txn.open_table(HASH_TO_MANIFEST_TABLE)?;
let mut keys = Vec::new();
let mut iter = hash_to_manifest_read.iter()?;
while let Some(entry) = iter.next() {
let (key, _) = entry?;
keys.push(key.value().to_string());
}
keys
};
// Then remove all keys
for key in keys_to_remove {
hash_to_manifest.remove(key.as_str())?;
}
}
}
write_txn.commit()?;
// Clear both tables
conn.execute("DELETE FROM obsoleted_packages", [])?;
conn.execute("DELETE FROM obsoleted_manifests", [])?;
Ok(())
}
/// Get the number of entries in the index
fn len(&self) -> Result<usize> {
// Begin a read transaction
let read_txn = self.db.begin_read()?;
let conn = Connection::open_with_flags(&self.db_path, OpenFlags::SQLITE_OPEN_READ_ONLY)?;
// Open the fmri_to_hash table
let fmri_to_hash = read_txn.open_table(FMRI_TO_METADATA_TABLE)?;
let count: i64 = conn.query_row(
"SELECT COUNT(*) FROM obsoleted_packages",
[],
|row| row.get(0),
)?;
// Count the entries
let mut count = 0;
let mut iter = fmri_to_hash.iter()?;
// Iterate through all entries and count them
while let Some(entry_result) = iter.next() {
// Just check if the entry exists, we don't need to access its value
entry_result?;
count += 1;
}
// Drop the iterator and table before returning
drop(iter);
drop(fmri_to_hash);
drop(read_txn);
Ok(count)
Ok(count as usize)
}
/// Check if the index is empty
@ -1033,8 +925,8 @@ impl ObsoletedPackageMetadata {
pub struct ObsoletedPackageManager {
/// Base path for obsoleted packages
base_path: PathBuf,
/// Index of obsoleted packages for faster lookups using redb
index: RwLock<RedbObsoletedPackageIndex>,
/// Index of obsoleted packages for faster lookups using SQLite
index: RwLock<SqliteObsoletedPackageIndex>,
}
impl ObsoletedPackageManager {
@ -1075,14 +967,14 @@ impl ObsoletedPackageManager {
let base_path = repo_path.as_ref().join("obsoleted");
let index = {
// Create or open the redb-based index
let redb_index =
RedbObsoletedPackageIndex::create_or_open(&base_path).unwrap_or_else(|e| {
// Log the error and create an empty redb index
error!("Failed to create or open redb-based index: {}", e);
RedbObsoletedPackageIndex::empty()
// Create or open the SQLite-based index
let sqlite_index =
SqliteObsoletedPackageIndex::create_or_open(&base_path).unwrap_or_else(|e| {
// Log the error and create an empty SQLite index
error!("Failed to create or open SQLite-based index: {}", e);
SqliteObsoletedPackageIndex::empty()
});
RwLock::new(redb_index)
RwLock::new(sqlite_index)
};
Self { base_path, index }

164
libips/src/repository/shard_sync.rs Normal file
View file

@ -0,0 +1,164 @@
// This Source Code Form is subject to the terms of
// the Mozilla Public License, v. 2.0. If a copy of the
// MPL was not distributed with this file, You can
// obtain one at https://mozilla.org/MPL/2.0/.
//! Client-side shard synchronization.
//!
//! Downloads catalog shards from the repository server and verifies their integrity.
use crate::repository::sqlite_catalog::{ShardIndex, ShardEntry};
use miette::Diagnostic;
use sha2::{Digest, Sha256};
use std::fs;
use std::path::{Path, PathBuf};
use thiserror::Error;
#[derive(Debug, Error, Diagnostic)]
#[error("Shard sync error: {message}")]
#[diagnostic(code(ips::shard_sync_error))]
pub struct ShardSyncError {
pub message: String,
}
impl ShardSyncError {
fn new(msg: impl Into<String>) -> Self {
Self {
message: msg.into(),
}
}
}
impl From<reqwest::Error> for ShardSyncError {
fn from(e: reqwest::Error) -> Self {
Self::new(format!("HTTP error: {}", e))
}
}
impl From<std::io::Error> for ShardSyncError {
fn from(e: std::io::Error) -> Self {
Self::new(format!("IO error: {}", e))
}
}
impl From<serde_json::Error> for ShardSyncError {
fn from(e: serde_json::Error) -> Self {
Self::new(format!("JSON error: {}", e))
}
}
/// Synchronize catalog shards from a repository origin.
///
/// Downloads the shard index from `{origin_url}/{publisher}/catalog/2/catalog.attrs`,
/// compares hashes with local copies, and downloads only changed shards.
///
/// # Arguments
/// * `publisher` - Publisher name
/// * `origin_url` - Repository origin URL (e.g., "https://pkg.example.com")
/// * `local_shard_dir` - Local directory to store shards
/// * `download_obsolete` - Whether to download obsolete.db (default: false)
pub fn sync_shards(
publisher: &str,
origin_url: &str,
local_shard_dir: &Path,
download_obsolete: bool,
) -> Result<(), ShardSyncError> {
// Ensure local directory exists
fs::create_dir_all(local_shard_dir)?;
// Fetch shard index
let index_url = format!("{}/{}/catalog/2/catalog.attrs", origin_url, publisher);
let client = reqwest::blocking::Client::new();
let response = client.get(&index_url).send()?;
if !response.status().is_success() {
return Err(ShardSyncError::new(format!(
"Failed to fetch shard index: HTTP {}",
response.status()
)));
}
let index: ShardIndex = response.json()?;
// List of shards to sync
let shards_to_sync = if download_obsolete {
vec!["active.db", "fts.db", "obsolete.db"]
} else {
vec!["active.db", "fts.db"]
};
// Download each shard if needed
for shard_name in shards_to_sync {
let Some(shard_entry) = index.shards.get(shard_name) else {
tracing::warn!("Shard {} not found in index", shard_name);
continue;
};
let local_path = local_shard_dir.join(shard_name);
// Check if local copy exists and matches hash
let needs_download = if local_path.exists() {
match compute_sha256(&local_path) {
Ok(local_hash) => local_hash != shard_entry.sha256,
Err(_) => true, // Error reading local file, re-download
}
} else {
true
};
if !needs_download {
tracing::debug!("Shard {} is up to date", shard_name);
continue;
}
// Download shard
tracing::info!("Downloading shard {} from {}", shard_name, origin_url);
let shard_url = format!(
"{}/{}/catalog/2/{}",
origin_url, publisher, &shard_entry.sha256
);
let mut response = client.get(&shard_url).send()?;
if !response.status().is_success() {
return Err(ShardSyncError::new(format!(
"Failed to download shard {}: HTTP {}",
shard_name,
response.status()
)));
}
// Write to temporary file
let temp_path = local_shard_dir.join(format!("{}.tmp", shard_name));
let mut file = fs::File::create(&temp_path)?;
response.copy_to(&mut file)?;
drop(file);
// Verify SHA-256
let downloaded_hash = compute_sha256(&temp_path)?;
if downloaded_hash != shard_entry.sha256 {
fs::remove_file(&temp_path)?;
return Err(ShardSyncError::new(format!(
"SHA-256 mismatch for {}: expected {}, got {}",
shard_name, shard_entry.sha256, downloaded_hash
)));
}
// Atomic rename
fs::rename(&temp_path, &local_path)?;
tracing::info!("Successfully downloaded {}", shard_name);
}
// Write local copy of index for future comparisons
let index_json = serde_json::to_string_pretty(&index)?;
fs::write(local_shard_dir.join("catalog.attrs"), index_json)?;
Ok(())
}
/// Compute SHA-256 hash of a file.
fn compute_sha256(path: &Path) -> Result<String, ShardSyncError> {
let bytes = fs::read(path)?;
let mut hasher = Sha256::new();
hasher.update(&bytes);
Ok(format!("{:x}", hasher.finalize()))
}

484
libips/src/repository/sqlite_catalog.rs Normal file
View file

@ -0,0 +1,484 @@
// This Source Code Form is subject to the terms of
// the Mozilla Public License, v. 2.0. If a copy of the
// MPL was not distributed with this file, You can
// obtain one at https://mozilla.org/MPL/2.0/.
//! SQLite catalog shard generation and population.
//!
//! This module defines all SQLite schemas used by the IPS system and provides
//! functions to build pre-built catalog shards for distribution via the
//! catalog/2 endpoint.
use crate::actions::Manifest;
use crate::fmri::Fmri;
use crate::repository::catalog::CatalogManager;
use base64::Engine as _;
use miette::Diagnostic;
use rusqlite::Connection;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::BTreeMap;
use std::fs;
use std::path::Path;
use thiserror::Error;
/// Schema for active.db - contains non-obsolete packages and their dependencies.
/// No manifest blobs stored; manifests are fetched from repository on demand.
pub const ACTIVE_SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS packages (
stem TEXT NOT NULL,
version TEXT NOT NULL,
publisher TEXT NOT NULL,
fmri TEXT GENERATED ALWAYS AS (
'pkg://' || publisher || '/' || stem || '@' || version
) STORED,
PRIMARY KEY (stem, version, publisher)
);
CREATE INDEX IF NOT EXISTS idx_packages_fmri ON packages(fmri);
CREATE INDEX IF NOT EXISTS idx_packages_stem ON packages(stem);
CREATE TABLE IF NOT EXISTS dependencies (
pkg_stem TEXT NOT NULL,
pkg_version TEXT NOT NULL,
pkg_publisher TEXT NOT NULL,
dep_type TEXT NOT NULL,
dep_stem TEXT NOT NULL,
dep_version TEXT,
PRIMARY KEY (pkg_stem, pkg_version, pkg_publisher, dep_type, dep_stem)
);
CREATE INDEX IF NOT EXISTS idx_deps_pkg ON dependencies(pkg_stem, pkg_version, pkg_publisher);
CREATE TABLE IF NOT EXISTS incorporate_locks (
stem TEXT NOT NULL PRIMARY KEY,
release TEXT NOT NULL
);
"#;
/// Schema for obsolete.db - client-side shard for obsoleted packages.
pub const OBSOLETE_SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS obsolete_packages (
publisher TEXT NOT NULL,
stem TEXT NOT NULL,
version TEXT NOT NULL,
fmri TEXT GENERATED ALWAYS AS (
'pkg://' || publisher || '/' || stem || '@' || version
) STORED,
PRIMARY KEY (publisher, stem, version)
);
CREATE INDEX IF NOT EXISTS idx_obsolete_fmri ON obsolete_packages(fmri);
"#;
/// Schema for fts.db - full-text search index.
pub const FTS_SCHEMA: &str = r#"
CREATE VIRTUAL TABLE IF NOT EXISTS package_search
USING fts5(stem, publisher, summary, description,
content='', tokenize='unicode61');
"#;
/// Schema for installed.db - tracks installed packages with manifest blobs.
pub const INSTALLED_SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS installed (
fmri TEXT NOT NULL PRIMARY KEY,
manifest BLOB NOT NULL
);
"#;
/// Schema for index.db (repository/obsoleted.rs) - server-side obsoleted package index.
pub const OBSOLETED_INDEX_SCHEMA: &str = r#"
CREATE TABLE IF NOT EXISTS obsoleted_packages (
fmri TEXT NOT NULL PRIMARY KEY,
publisher TEXT NOT NULL,
stem TEXT NOT NULL,
version TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'obsolete',
obsolescence_date TEXT NOT NULL,
deprecation_message TEXT,
obsoleted_by TEXT,
metadata_version INTEGER NOT NULL DEFAULT 1,
content_hash TEXT NOT NULL
);
CREATE INDEX IF NOT EXISTS idx_obsidx_stem ON obsoleted_packages(stem);
CREATE TABLE IF NOT EXISTS obsoleted_manifests (
content_hash TEXT NOT NULL PRIMARY KEY,
manifest TEXT NOT NULL
);
"#;
#[derive(Debug, Error, Diagnostic)]
#[error("Shard building error: {message}")]
#[diagnostic(code(ips::shard_build_error))]
pub struct ShardBuildError {
pub message: String,
}
impl ShardBuildError {
fn new(msg: impl Into<String>) -> Self {
Self {
message: msg.into(),
}
}
}
impl From<rusqlite::Error> for ShardBuildError {
fn from(e: rusqlite::Error) -> Self {
Self::new(format!("SQLite error: {}", e))
}
}
impl From<std::io::Error> for ShardBuildError {
fn from(e: std::io::Error) -> Self {
Self::new(format!("IO error: {}", e))
}
}
impl From<serde_json::Error> for ShardBuildError {
fn from(e: serde_json::Error) -> Self {
Self::new(format!("JSON error: {}", e))
}
}
/// Shard metadata entry in catalog.attrs.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ShardEntry {
pub sha256: String,
pub size: u64,
#[serde(rename = "last-modified")]
pub last_modified: String,
}
/// Shard index JSON structure for catalog/2/catalog.attrs.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct ShardIndex {
pub version: u32,
pub created: String,
#[serde(rename = "last-modified")]
pub last_modified: String,
#[serde(rename = "package-count")]
pub package_count: usize,
#[serde(rename = "package-version-count")]
pub package_version_count: usize,
pub shards: BTreeMap<String, ShardEntry>,
}
/// Build catalog shards from JSON catalog parts.
///
/// Reads catalog parts from `catalog_parts_dir`, generates active.db, obsolete.db,
/// and fts.db, writes them to `output_dir`, and creates catalog.attrs index.
pub fn build_shards(
catalog_parts_dir: &Path,
publisher: &str,
output_dir: &Path,
) -> Result<(), ShardBuildError> {
// Create temp directory for shard generation
fs::create_dir_all(output_dir)?;
let temp_dir = output_dir.join(".tmp");
fs::create_dir_all(&temp_dir)?;
// Create shard databases
let active_path = temp_dir.join("active.db");
let obsolete_path = temp_dir.join("obsolete.db");
let fts_path = temp_dir.join("fts.db");
let mut active_conn = Connection::open(&active_path)?;
let mut obsolete_conn = Connection::open(&obsolete_path)?;
let mut fts_conn = Connection::open(&fts_path)?;
// Execute schemas
active_conn.execute_batch(ACTIVE_SCHEMA)?;
obsolete_conn.execute_batch(OBSOLETE_SCHEMA)?;
fts_conn.execute_batch(FTS_SCHEMA)?;
// Read catalog parts
let catalog_manager = CatalogManager::new(catalog_parts_dir, publisher)
.map_err(|e| ShardBuildError::new(format!("Failed to create catalog manager: {}", e)))?;
let mut package_count = 0usize;
let mut package_version_count = 0usize;
// Begin transactions for batch inserts
let active_tx = active_conn.transaction()?;
let obsolete_tx = obsolete_conn.transaction()?;
let fts_tx = fts_conn.transaction()?;
{
let mut insert_pkg = active_tx.prepare(
"INSERT OR REPLACE INTO packages (stem, version, publisher) VALUES (?1, ?2, ?3)",
)?;
let mut insert_dep = active_tx.prepare(
"INSERT OR REPLACE INTO dependencies (pkg_stem, pkg_version, pkg_publisher, dep_type, dep_stem, dep_version) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
)?;
let mut insert_obs = obsolete_tx.prepare(
"INSERT OR REPLACE INTO obsolete_packages (publisher, stem, version) VALUES (?1, ?2, ?3)",
)?;
let mut insert_fts = fts_tx.prepare(
"INSERT INTO package_search (stem, publisher, summary, description) VALUES (?1, ?2, ?3, ?4)",
)?;
// Iterate catalog parts
let part_names: Vec<String> = catalog_manager.attrs().parts.keys().cloned().collect();
for part_name in part_names {
let part_path = catalog_parts_dir.join(&part_name);
// Load the CatalogPart
let part = crate::repository::catalog::CatalogPart::load(&part_path)
.map_err(|e| ShardBuildError::new(format!("Failed to load catalog part: {}", e)))?;
// Iterate through publishers in the catalog part
for (part_publisher, stems) in &part.packages {
// Only process packages for the requested publisher
if part_publisher != publisher {
continue;
}
// Iterate through package stems
for (pkg_name, versions) in stems {
// Iterate through versions
for version_entry in versions {
let pkg_version = &version_entry.version;
// Build a minimal manifest from the actions
let mut manifest = Manifest::new();
// Parse actions if available
if let Some(actions) = &version_entry.actions {
for action_str in actions {
if action_str.starts_with("set ") {
// Parse "set name=key value=val" format
let parts: Vec<&str> = action_str.split_whitespace().collect();
if parts.len() >= 3 {
if let Some(name_part) = parts.get(1) {
if let Some(key) = name_part.strip_prefix("name=") {
if let Some(value_part) = parts.get(2) {
if let Some(mut value) = value_part.strip_prefix("value=") {
// Remove quotes
value = value.trim_matches('"');
let mut attr = crate::actions::Attr::default();
attr.key = key.to_string();
attr.values = vec![value.to_string()];
manifest.attributes.push(attr);
}
}
}
}
}
} else if action_str.starts_with("depend ") {
// Parse "depend fmri=... type=..." format
let mut dep = crate::actions::Dependency::default();
for part in action_str.split_whitespace().skip(1) {
if let Some((k, v)) = part.split_once('=') {
match k {
"fmri" => {
if let Ok(f) = crate::fmri::Fmri::parse(v) {
dep.fmri = Some(f);
}
}
"type" => {
dep.dependency_type = v.to_string();
}
_ => {}
}
}
}
if dep.fmri.is_some() && !dep.dependency_type.is_empty() {
manifest.dependencies.push(dep);
}
}
}
}
// Determine if obsolete
let is_obsolete = crate::image::catalog::is_package_obsolete(&manifest);
// Count all package versions
package_version_count += 1;
// Obsolete packages go only to obsolete.db, non-obsolete go to active.db
if is_obsolete {
insert_obs.execute(rusqlite::params![publisher, pkg_name, pkg_version])?;
} else {
// Insert into packages table (active.db)
insert_pkg.execute(rusqlite::params![pkg_name, pkg_version, publisher])?;
// Extract and insert dependencies
for dep in &manifest.dependencies {
if dep.dependency_type == "require" || dep.dependency_type == "incorporate" {
if let Some(dep_fmri) = &dep.fmri {
let dep_stem = dep_fmri.stem();
let dep_version = dep_fmri.version.as_ref().map(|v| v.to_string());
insert_dep.execute(rusqlite::params![
pkg_name,
pkg_version,
publisher,
&dep.dependency_type,
dep_stem,
dep_version
])?;
}
}
}
}
// Extract summary and description for FTS
let summary = manifest
.attributes
.iter()
.find(|a| a.key == "pkg.summary")
.and_then(|a| a.values.first())
.map(|s| s.as_str())
.unwrap_or("");
let description = manifest
.attributes
.iter()
.find(|a| a.key == "pkg.description")
.and_then(|a| a.values.first())
.map(|s| s.as_str())
.unwrap_or("");
insert_fts.execute(rusqlite::params![
pkg_name,
publisher,
summary,
description
])?;
}
}
}
}
}
// Commit transactions
active_tx.commit()?;
obsolete_tx.commit()?;
fts_tx.commit()?;
// Count unique packages (stems)
let count: i64 = active_conn.query_row(
"SELECT COUNT(DISTINCT stem) FROM packages",
[],
|row| row.get(0),
)?;
package_count = count as usize;
// Close connections
drop(active_conn);
drop(obsolete_conn);
drop(fts_conn);
// Compute SHA-256 hashes and build index
let mut shards = BTreeMap::new();
let now = chrono::Utc::now().format("%Y%m%dT%H%M%SZ").to_string();
for (name, path) in [
("active.db", &active_path),
("obsolete.db", &obsolete_path),
("fts.db", &fts_path),
] {
let bytes = fs::read(path)?;
let mut hasher = Sha256::new();
hasher.update(&bytes);
let hash = format!("{:x}", hasher.finalize());
let size = bytes.len() as u64;
shards.insert(
name.to_string(),
ShardEntry {
sha256: hash.clone(),
size,
last_modified: now.clone(),
},
);
// Copy shard to output directory with both original name and hash-based name
// Keep original name for client-side use (e.g., active.db, obsolete.db)
let named_path = output_dir.join(name);
fs::copy(path, &named_path)?;
// Also copy to hash-based name for content-addressed server distribution
let hash_path = output_dir.join(&hash);
fs::copy(path, &hash_path)?;
}
// Write catalog.attrs
let index = ShardIndex {
version: 2,
created: now.clone(),
last_modified: now,
package_count,
package_version_count,
shards,
};
let index_json = serde_json::to_string_pretty(&index)?;
fs::write(output_dir.join("catalog.attrs"), index_json)?;
// Clean up temp directory
fs::remove_dir_all(&temp_dir).ok();
Ok(())
}
/// Helper function for tests: populate active.db with a single package.
/// Creates tables if absent (idempotent).
pub fn populate_active_db(
db_path: &Path,
fmri: &Fmri,
manifest: &Manifest,
) -> Result<(), ShardBuildError> {
let mut conn = Connection::open(db_path)?;
conn.execute_batch(ACTIVE_SCHEMA)?;
let tx = conn.transaction()?;
{
tx.execute(
"INSERT OR REPLACE INTO packages (stem, version, publisher) VALUES (?1, ?2, ?3)",
rusqlite::params![
fmri.stem(),
fmri.version(),
fmri.publisher.as_deref().unwrap_or("")
],
)?;
for dep in &manifest.dependencies {
if dep.dependency_type == "require" || dep.dependency_type == "incorporate" {
if let Some(dep_fmri) = &dep.fmri {
let dep_stem = dep_fmri.stem();
let dep_version = dep_fmri.version.as_ref().map(|v| v.to_string());
tx.execute(
"INSERT OR REPLACE INTO dependencies (pkg_stem, pkg_version, pkg_publisher, dep_type, dep_stem, dep_version) VALUES (?1, ?2, ?3, ?4, ?5, ?6)",
rusqlite::params![
fmri.stem(),
fmri.version(),
fmri.publisher.as_deref().unwrap_or(""),
&dep.dependency_type,
dep_stem,
dep_version
],
)?;
}
}
}
}
tx.commit()?;
Ok(())
}
/// Helper function for tests: mark a package as obsolete in obsolete.db.
/// Creates tables if absent (idempotent).
pub fn populate_obsolete_db(db_path: &Path, fmri: &Fmri) -> Result<(), ShardBuildError> {
let mut conn = Connection::open(db_path)?;
conn.execute_batch(OBSOLETE_SCHEMA)?;
conn.execute(
"INSERT OR REPLACE INTO obsolete_packages (publisher, stem, version) VALUES (?1, ?2, ?3)",
rusqlite::params![
fmri.publisher.as_deref().unwrap_or(""),
fmri.stem(),
fmri.version()
],
)?;
Ok(())
}
// Note: compress_json_lz4, decode_manifest_bytes, and is_package_obsolete
// are available as pub(crate) in crate::image::catalog and can be used
// within libips but not re-exported.

365
libips/src/solver/mod.rs
View file

@ -18,55 +18,23 @@
use miette::Diagnostic;
// Begin resolvo wiring imports (names discovered by compiler)
// We start broad and refine with compiler guidance.
use lz4::Decoder as Lz4Decoder;
use redb::{ReadableDatabase, ReadableTable};
use resolvo::{
self, Candidates, Condition, ConditionId, ConditionalRequirement,
Dependencies as RDependencies, DependencyProvider, HintDependenciesAvailable, Interner,
KnownDependencies, Mapping, NameId, Problem as RProblem, SolvableId, Solver as RSolver,
SolverCache, StringId, UnsolvableOrCancelled, VersionSetId, VersionSetUnionId,
};
use rusqlite::{Connection, OpenFlags};
use std::cell::RefCell;
use std::collections::{BTreeMap, HashMap};
use std::fmt::Display;
use std::io::{Cursor, Read};
use thiserror::Error;
use crate::actions::Manifest;
use crate::image::catalog::{CATALOG_TABLE, INCORPORATE_TABLE};
// Public advice API lives in a sibling module
pub mod advice;
// Local helpers to decode manifest bytes stored in catalog DB (JSON or LZ4-compressed JSON)
fn is_likely_json_local(bytes: &[u8]) -> bool {
let mut i = 0;
while i < bytes.len() && matches!(bytes[i], b' ' | b'\n' | b'\r' | b'\t') {
i += 1;
}
if i >= bytes.len() {
return false;
}
matches!(bytes[i], b'{' | b'[')
}
fn decode_manifest_bytes_local(bytes: &[u8]) -> Result<Manifest, serde_json::Error> {
if is_likely_json_local(bytes) {
return serde_json::from_slice::<Manifest>(bytes);
}
// Try LZ4; on failure, fall back to JSON attempt
if let Ok(mut dec) = Lz4Decoder::new(Cursor::new(bytes)) {
let mut out = Vec::new();
if dec.read_to_end(&mut out).is_ok() {
if let Ok(m) = serde_json::from_slice::<Manifest>(&out) {
return Ok(m);
}
}
}
// Fallback to JSON parse of original bytes
serde_json::from_slice::<Manifest>(bytes)
}
#[derive(Clone, Debug)]
struct PkgCand {
#[allow(dead_code)]
@ -85,11 +53,8 @@ enum VersionSetKind {
struct IpsProvider<'a> {
image: &'a Image,
// Persistent database handles and read transactions for catalog/obsoleted
_catalog_db: redb::Database,
catalog_tx: redb::ReadTransaction,
_obsoleted_db: redb::Database,
_obsoleted_tx: redb::ReadTransaction,
// SQLite connection to active.db (catalog), opened read-only
catalog_conn: Connection,
// interner storages
names: Mapping<NameId, String>,
name_by_str: BTreeMap<String, NameId>,
@ -108,24 +73,21 @@ use crate::image::Image;
impl<'a> IpsProvider<'a> {
fn new(image: &'a Image) -> Result<Self, SolverError> {
// Open databases and keep read transactions alive for the provider lifetime
let catalog_db = redb::Database::open(image.catalog_db_path())
// Open active.db (catalog) read-only with WAL mode for better concurrency
let catalog_conn = Connection::open_with_flags(
&image.active_db_path(),
OpenFlags::SQLITE_OPEN_READ_ONLY,
)
.map_err(|e| SolverError::new(format!("open catalog db: {}", e)))?;
let catalog_tx = catalog_db
.begin_read()
.map_err(|e| SolverError::new(format!("begin read catalog db: {}", e)))?;
let obsoleted_db = redb::Database::open(image.obsoleted_db_path())
.map_err(|e| SolverError::new(format!("open obsoleted db: {}", e)))?;
let obsoleted_tx = obsoleted_db
.begin_read()
.map_err(|e| SolverError::new(format!("begin read obsoleted db: {}", e)))?;
// Enable WAL mode for better concurrency (ignored if already set)
catalog_conn
.pragma_update(None, "journal_mode", "WAL")
.ok();
let mut prov = IpsProvider {
image,
_catalog_db: catalog_db,
catalog_tx,
_obsoleted_db: obsoleted_db,
_obsoleted_tx: obsoleted_tx,
catalog_conn,
names: Mapping::default(),
name_by_str: BTreeMap::new(),
strings: Mapping::default(),
@ -141,30 +103,52 @@ impl<'a> IpsProvider<'a> {
}
fn build_index(&mut self) -> Result<(), SolverError> {
use crate::image::catalog::CATALOG_TABLE;
// Iterate catalog table and build in-memory index of non-obsolete candidates
let table = self
.catalog_tx
.open_table(CATALOG_TABLE)
.map_err(|e| SolverError::new(format!("open catalog table: {}", e)))?;
// Query packages table directly - no manifest decoding needed
// Use a scope to ensure stmt is dropped before we start mutating self
let collected_rows: Vec<Result<(String, String, String), rusqlite::Error>> = {
let mut stmt = self
.catalog_conn
.prepare("SELECT stem, version, publisher FROM packages")
.map_err(|e| SolverError::new(format!("prepare packages query: {}", e)))?;
let rows = stmt
.query_map([], |row| {
Ok((
row.get::<_, String>(0)?,
row.get::<_, String>(1)?,
row.get::<_, String>(2)?,
))
})
.map_err(|e| SolverError::new(format!("query packages: {}", e)))?;
// Collect all rows into a Vec to avoid holding a borrow on the connection
rows.collect()
};
// Temporary map: stem string -> Vec<Fmri>
let mut by_stem: BTreeMap<String, Vec<Fmri>> = BTreeMap::new();
for entry in table
.iter()
.map_err(|e| SolverError::new(format!("iterate catalog table: {}", e)))?
{
let (k, v) =
entry.map_err(|e| SolverError::new(format!("read catalog entry: {}", e)))?;
let key = k.value(); // stem@version
// Try to decode manifest and extract full FMRI (including publisher)
let mut pushed = false;
if let Ok(manifest) = decode_manifest_bytes_local(v.value()) {
if let Some(attr) = manifest.attributes.iter().find(|a| a.key == "pkg.fmri") {
if let Some(fmri_str) = attr.values.first() {
if let Ok(mut fmri) = Fmri::parse(fmri_str) {
// Ensure publisher is present; if missing/empty, use image default publisher
for row_result in collected_rows {
let (stem, version, publisher) = row_result
.map_err(|e| SolverError::new(format!("read package row: {}", e)))?;
// Parse version
let ver_obj = crate::fmri::Version::parse(&version).ok();
// Build FMRI with publisher, stem, and version
let mut fmri = if let Some(v) = ver_obj {
Fmri::with_publisher(&publisher, &stem, Some(v))
} else {
// No parsable version; still record a minimal FMRI without version
Fmri::with_publisher(&publisher, &stem, None)
};
// Normalize: empty publisher string -> None
if fmri.publisher.as_deref() == Some("") {
fmri.publisher = None;
}
// If publisher is still None/empty, try using image default publisher
let missing_pub = fmri
.publisher
.as_deref()
@ -175,46 +159,14 @@ impl<'a> IpsProvider<'a> {
fmri.publisher = Some(defp.name.clone());
}
}
by_stem
.entry(fmri.stem().to_string())
.or_default()
.push(fmri);
pushed = true;
}
}
}
}
// Fallback: derive FMRI from catalog key if we couldn't push from manifest
if !pushed {
if let Some((stem, ver_str)) = key.split_once('@') {
let ver_obj = crate::fmri::Version::parse(ver_str).ok();
// Prefer default publisher if configured; else leave None by constructing and then setting publisher
let mut fmri = if let Some(v) = ver_obj.clone() {
if let Ok(defp) = self.image.default_publisher() {
Fmri::with_publisher(&defp.name, stem, Some(v))
} else {
Fmri::with_version(stem, v)
}
} else {
// No parsable version; still record a minimal FMRI without version
if let Ok(defp) = self.image.default_publisher() {
Fmri::with_publisher(&defp.name, stem, None)
} else {
Fmri::with_publisher("", stem, None)
}
};
// Normalize: empty publisher string -> None
if fmri.publisher.as_deref() == Some("") {
fmri.publisher = None;
}
by_stem.entry(stem.to_string()).or_default().push(fmri);
}
}
by_stem.entry(stem).or_default().push(fmri);
}
// Intern and populate solvables per stem
for (stem, mut fmris) in by_stem {
// Collect into Vec to avoid borrow checker issues with mutating self while iterating
let stems_and_fmris: Vec<(String, Vec<Fmri>)> = by_stem.into_iter().collect();
for (stem, mut fmris) in stems_and_fmris {
let name_id = self.intern_name(&stem);
// Sort fmris newest-first using IPS ordering
fmris.sort_by(|a, b| version_order_desc(a, b));
@ -254,22 +206,13 @@ impl<'a> IpsProvider<'a> {
}
fn lookup_incorporated_release(&self, stem: &str) -> Option<String> {
if let Ok(table) = self.catalog_tx.open_table(INCORPORATE_TABLE) {
if let Ok(Some(rel)) = table.get(stem) {
return Some(String::from_utf8_lossy(rel.value()).to_string());
}
}
None
}
fn read_manifest_from_catalog(&self, fmri: &Fmri) -> Option<Manifest> {
let key = format!("{}@{}", fmri.stem(), fmri.version());
if let Ok(table) = self.catalog_tx.open_table(CATALOG_TABLE) {
if let Ok(Some(bytes)) = table.get(key.as_str()) {
return decode_manifest_bytes_local(bytes.value()).ok();
}
}
None
self.catalog_conn
.query_row(
"SELECT release FROM incorporate_locks WHERE stem = ?1",
rusqlite::params![stem],
|row| row.get(0),
)
.ok()
}
}
@ -516,9 +459,31 @@ impl<'a> DependencyProvider for IpsProvider<'a> {
async fn get_dependencies(&self, solvable: SolvableId) -> RDependencies {
let pkg = self.solvables.get(solvable).unwrap();
let fmri = &pkg.fmri;
let manifest_opt = self.read_manifest_from_catalog(fmri);
let Some(manifest) = manifest_opt else {
return RDependencies::Known(KnownDependencies::default());
// Query dependencies table directly instead of decoding manifest
let mut stmt = match self.catalog_conn.prepare(
"SELECT dep_stem, dep_version FROM dependencies
WHERE pkg_stem = ?1 AND pkg_version = ?2 AND pkg_publisher = ?3 AND dep_type = 'require'"
) {
Ok(s) => s,
Err(_) => return RDependencies::Known(KnownDependencies::default()),
};
let parent_stem = fmri.stem();
let parent_version = fmri.version();
let parent_publisher = fmri.publisher.as_deref().unwrap_or("");
let rows = match stmt.query_map(
rusqlite::params![parent_stem, parent_version, parent_publisher],
|row| {
Ok((
row.get::<_, String>(0)?, // dep_stem
row.get::<_, Option<String>>(1)?, // dep_version
))
},
) {
Ok(r) => r,
Err(_) => return RDependencies::Known(KnownDependencies::default()),
};
// Build requirements for "require" deps
@ -526,19 +491,24 @@ impl<'a> DependencyProvider for IpsProvider<'a> {
let parent_branch = fmri.version.as_ref().and_then(|v| v.branch.clone());
let parent_pub = fmri.publisher.as_deref();
for d in manifest
.dependencies
.iter()
.filter(|d| d.dependency_type == "require")
{
if let Some(df) = &d.fmri {
let stem = df.stem().to_string();
let Some(child_name_id) = self.name_by_str.get(&stem).copied() else {
for row_result in rows {
let (dep_stem, dep_version_str) = match row_result {
Ok(r) => r,
Err(_) => continue,
};
let Some(child_name_id) = self.name_by_str.get(&dep_stem).copied() else {
// If the dependency name isn't present in the catalog index, skip it
continue;
};
// Parse dep_version to extract release component
let dep_version = dep_version_str
.as_ref()
.and_then(|s| crate::fmri::Version::parse(s).ok());
// Create version set by release (from dep expr) and branch (from parent)
let vs_kind = match (&df.version, &parent_branch) {
let vs_kind = match (&dep_version, &parent_branch) {
(Some(ver), Some(branch)) => VersionSetKind::ReleaseAndBranch {
release: ver.release.clone(),
branch: branch.clone(),
@ -557,7 +527,7 @@ impl<'a> DependencyProvider for IpsProvider<'a> {
.entry(child_name_id)
.or_insert(order);
}
}
RDependencies::Known(KnownDependencies {
requirements: reqs,
constrains: vec![],
@ -820,18 +790,6 @@ pub fn resolve_install(
}
name_to_fmris.insert(*name_id, v);
}
// Snapshot: Catalog manifest cache keyed by stem@version for all candidates
let mut key_to_manifest: HashMap<String, Manifest> = HashMap::new();
for fmris in name_to_fmris.values() {
for fmri in fmris {
let key = format!("{}@{}", fmri.stem(), fmri.version());
if !key_to_manifest.contains_key(&key) {
if let Some(man) = provider.read_manifest_from_catalog(fmri) {
key_to_manifest.insert(key, man);
}
}
}
}
// Run the solver
let roots_for_err: Vec<Constraint> = root_names.iter().map(|(_, c)| c.clone()).collect();
@ -864,15 +822,10 @@ pub fn resolve_install(
let mut plan = InstallPlan::default();
for sid in solution_ids {
if let Some(fmri) = sid_to_fmri.get(&sid).cloned() {
// Prefer repository manifest; fallback to preloaded catalog snapshot, then image catalog
let key = format!("{}@{}", fmri.stem(), fmri.version());
// Fetch manifest from repository or catalog cache
let manifest = match image_ref.get_manifest_from_repository(&fmri) {
Ok(m) => m,
Err(repo_err) => {
if let Some(m) = key_to_manifest.get(&key).cloned() {
m
} else {
match image_ref.get_manifest_from_catalog(&fmri) {
Err(repo_err) => match image_ref.get_manifest_from_catalog(&fmri) {
Ok(Some(m)) => m,
_ => {
return Err(SolverError::new(format!(
@ -880,9 +833,7 @@ pub fn resolve_install(
fmri, repo_err
)));
}
}
}
}
},
};
plan.reasons.push(format!("selected {} via solver", fmri));
plan.add.push(ResolvedPkg { fmri, manifest });
@ -933,8 +884,7 @@ mod solver_integration_tests {
use crate::actions::Dependency;
use crate::fmri::Version;
use crate::image::ImageType;
use crate::image::catalog::{CATALOG_TABLE, OBSOLETED_TABLE};
use redb::Database;
use crate::repository::sqlite_catalog;
use tempfile::tempdir;
fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version {
@ -970,34 +920,13 @@ mod solver_integration_tests {
}
fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) {
let db = Database::open(image.catalog_db_path()).expect("open catalog db");
let tx = db.begin_write().expect("begin write");
{
let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table");
let key = format!("{}@{}", fmri.stem(), fmri.version());
let val = serde_json::to_vec(manifest).expect("serialize manifest");
table
.insert(key.as_str(), val.as_slice())
.expect("insert manifest");
}
tx.commit().expect("commit");
sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest)
.expect("populate active db");
}
fn mark_obsolete(image: &Image, fmri: &Fmri) {
let db = Database::open(image.obsoleted_db_path()).expect("open obsoleted db");
let tx = db.begin_write().expect("begin write");
{
let mut table = tx
.open_table(OBSOLETED_TABLE)
.expect("open obsoleted table");
let key = fmri.to_string();
// store empty value
let empty: Vec<u8> = Vec::new();
table
.insert(key.as_str(), empty.as_slice())
.expect("insert obsolete");
}
tx.commit().expect("commit");
sqlite_catalog::populate_obsolete_db(&image.obsolete_db_path(), fmri)
.expect("populate obsolete db");
}
fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image {
@ -1322,8 +1251,7 @@ mod solver_error_message_tests {
use crate::actions::{Dependency, Manifest};
use crate::fmri::{Fmri, Version};
use crate::image::ImageType;
use crate::image::catalog::CATALOG_TABLE;
use redb::Database;
use crate::repository::sqlite_catalog;
fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version {
let mut v = Version::new(release);
@ -1354,17 +1282,8 @@ mod solver_error_message_tests {
}
fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) {
let db = Database::open(image.catalog_db_path()).expect("open catalog db");
let tx = db.begin_write().expect("begin write");
{
let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table");
let key = format!("{}@{}", fmri.stem(), fmri.version());
let val = serde_json::to_vec(manifest).expect("serialize manifest");
table
.insert(key.as_str(), val.as_slice())
.expect("insert manifest");
}
tx.commit().expect("commit");
sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest)
.expect("populate active db");
}
#[test]
@ -1427,8 +1346,7 @@ mod incorporate_lock_tests {
use crate::actions::Dependency;
use crate::fmri::Version;
use crate::image::ImageType;
use crate::image::catalog::CATALOG_TABLE;
use redb::Database;
use crate::repository::sqlite_catalog;
use tempfile::tempdir;
fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version {
@ -1447,17 +1365,8 @@ mod incorporate_lock_tests {
}
fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) {
let db = Database::open(image.catalog_db_path()).expect("open catalog db");
let tx = db.begin_write().expect("begin write");
{
let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table");
let key = format!("{}@{}", fmri.stem(), fmri.version());
let val = serde_json::to_vec(manifest).expect("serialize manifest");
table
.insert(key.as_str(), val.as_slice())
.expect("insert manifest");
}
tx.commit().expect("commit");
sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest)
.expect("populate active db");
}
fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image {
@ -1638,8 +1547,7 @@ mod composite_release_tests {
use crate::actions::{Dependency, Manifest};
use crate::fmri::{Fmri, Version};
use crate::image::ImageType;
use crate::image::catalog::CATALOG_TABLE;
use redb::Database;
use crate::repository::sqlite_catalog;
fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version {
let mut v = Version::new(release);
@ -1657,17 +1565,8 @@ mod composite_release_tests {
}
fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) {
let db = Database::open(image.catalog_db_path()).expect("open catalog db");
let tx = db.begin_write().expect("begin write");
{
let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table");
let key = format!("{}@{}", fmri.stem(), fmri.version());
let val = serde_json::to_vec(manifest).expect("serialize manifest");
table
.insert(key.as_str(), val.as_slice())
.expect("insert manifest");
}
tx.commit().expect("commit");
sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest)
.expect("populate active db");
}
fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image {
@ -1772,8 +1671,7 @@ mod circular_dependency_tests {
use crate::actions::Dependency;
use crate::fmri::{Fmri, Version};
use crate::image::ImageType;
use crate::image::catalog::CATALOG_TABLE;
use redb::Database;
use crate::repository::sqlite_catalog;
use std::collections::HashSet;
fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version {
@ -1809,17 +1707,8 @@ mod circular_dependency_tests {
}
fn write_manifest_to_catalog(image: &Image, fmri: &Fmri, manifest: &Manifest) {
let db = Database::open(image.catalog_db_path()).expect("open catalog db");
let tx = db.begin_write().expect("begin write");
{
let mut table = tx.open_table(CATALOG_TABLE).expect("open catalog table");
let key = format!("{}@{}", fmri.stem(), fmri.version());
let val = serde_json::to_vec(manifest).expect("serialize manifest");
table
.insert(key.as_str(), val.as_slice())
.expect("insert manifest");
}
tx.commit().expect("commit");
sqlite_catalog::populate_active_db(&image.active_db_path(), fmri, manifest)
.expect("populate active db");
}
fn make_image_with_publishers(pubs: &[(&str, bool)]) -> Image {

6
pkg6depotd/Cargo.toml
View file

@ -31,6 +31,8 @@ serde_json = "1.0"
dirs = "6"
nix = { version = "0.30", features = ["signal", "process", "user", "fs"] }
sha1 = "0.10"
sha2 = "0.10"
rusqlite = { version = "0.31", default-features = false }
chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
flate2 = "1"
httpdate = "1"
@ -55,3 +57,7 @@ reqwest = { version = "0.12", features = ["blocking", "json"] }
assert_cmd = "2"
predicates = "3"
tempfile = "3"
[features]
default = ["bundled-sqlite"]
bundled-sqlite = ["rusqlite/bundled"]

1
pkg6depotd/src/http/handlers/mod.rs
View file

@ -4,4 +4,5 @@ pub mod info;
pub mod manifest;
pub mod publisher;
pub mod search;
pub mod shard;
pub mod versions;

127
pkg6depotd/src/http/handlers/shard.rs Normal file
View file

@ -0,0 +1,127 @@
use crate::errors::DepotError;
use crate::repo::DepotRepo;
use axum::extract::{Path, Request, State};
use axum::http::header;
use axum::response::{IntoResponse, Response};
use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;
use std::fs;
use std::sync::Arc;
use tower::ServiceExt;
use tower_http::services::ServeFile;
/// Shard metadata entry in catalog.attrs.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct ShardEntry {
sha256: String,
size: u64,
#[serde(rename = "last-modified")]
last_modified: String,
}
/// Shard index JSON structure for catalog/2/catalog.attrs.
#[derive(Debug, Clone, Serialize, Deserialize)]
struct ShardIndex {
version: u32,
created: String,
#[serde(rename = "last-modified")]
last_modified: String,
#[serde(rename = "package-count")]
package_count: usize,
#[serde(rename = "package-version-count")]
package_version_count: usize,
shards: BTreeMap<String, ShardEntry>,
}
/// GET /{publisher}/catalog/2/catalog.attrs
pub async fn get_shard_index(
State(repo): State<Arc<DepotRepo>>,
Path(publisher): Path<String>,
) -> Result<Response, DepotError> {
let shard_dir = repo.shard_dir(&publisher);
let index_path = shard_dir.join("catalog.attrs");
if !index_path.exists() {
return Err(DepotError::Repo(
libips::repository::RepositoryError::NotFound(
"catalog.attrs not found - shards not yet built".to_string(),
),
));
}
let content = fs::read_to_string(&index_path)
.map_err(|e| DepotError::Server(format!("Failed to read catalog.attrs: {}", e)))?;
Ok(([(header::CONTENT_TYPE, "application/json")], content).into_response())
}
/// GET /{publisher}/catalog/2/{sha256}
pub async fn get_shard_blob(
State(repo): State<Arc<DepotRepo>>,
Path((publisher, sha256)): Path<(String, String)>,
req: Request,
) -> Result<Response, DepotError> {
let shard_dir = repo.shard_dir(&publisher);
let index_path = shard_dir.join("catalog.attrs");
if !index_path.exists() {
return Err(DepotError::Repo(
libips::repository::RepositoryError::NotFound(
"catalog.attrs not found - shards not yet built".to_string(),
),
));
}
// Read index to validate hash
let index_content = fs::read_to_string(&index_path)
.map_err(|e| DepotError::Server(format!("Failed to read catalog.attrs: {}", e)))?;
let index: ShardIndex = serde_json::from_str(&index_content)
.map_err(|e| DepotError::Server(format!("Failed to parse catalog.attrs: {}", e)))?;
// Find which shard file corresponds to this hash
let mut shard_path: Option<std::path::PathBuf> = None;
for (name, entry) in &index.shards {
if entry.sha256 == sha256 {
shard_path = Some(shard_dir.join(&sha256));
break;
}
}
let Some(path) = shard_path else {
return Err(DepotError::Repo(
libips::repository::RepositoryError::NotFound(format!(
"Shard with hash {} not found",
sha256
)),
));
};
if !path.exists() {
return Err(DepotError::Repo(
libips::repository::RepositoryError::NotFound(format!(
"Shard file {} not found on disk",
sha256
)),
));
}
// Serve the file
let service = ServeFile::new(path);
let result = service.oneshot(req).await;
match result {
Ok(mut res) => {
// Add cache headers - content is content-addressed and immutable
res.headers_mut().insert(
header::CONTENT_TYPE,
header::HeaderValue::from_static("application/octet-stream"),
);
res.headers_mut().insert(
header::CACHE_CONTROL,
header::HeaderValue::from_static("public, immutable, max-age=86400"),
);
Ok(res.into_response())
}
Err(e) => Err(DepotError::Server(e.to_string())),
}
}

2
pkg6depotd/src/http/handlers/versions.rs
View file

@ -68,7 +68,7 @@ pub async fn get_versions() -> impl IntoResponse {
},
SupportedOperation {
op: Operation::Catalog,
versions: vec![1],
versions: vec![1, 2],
},
SupportedOperation {
op: Operation::Manifest,

10
pkg6depotd/src/http/routes.rs
View file

@ -1,5 +1,5 @@
use crate::http::admin;
use crate::http::handlers::{catalog, file, info, manifest, publisher, search, versions};
use crate::http::handlers::{catalog, file, info, manifest, publisher, search, shard, versions};
use crate::repo::DepotRepo;
use axum::{
Router,
@ -16,6 +16,14 @@ pub fn app_router(state: Arc<DepotRepo>) -> Router {
"/{publisher}/catalog/1/{filename}",
get(catalog::get_catalog_v1).head(catalog::get_catalog_v1),
)
.route(
"/{publisher}/catalog/2/catalog.attrs",
get(shard::get_shard_index).head(shard::get_shard_index),
)
.route(
"/{publisher}/catalog/2/{sha256}",
get(shard::get_shard_blob).head(shard::get_shard_blob),
)
.route(
"/{publisher}/manifest/0/{fmri}",
get(manifest::get_manifest).head(manifest::get_manifest),

7
pkg6depotd/src/repo.rs
View file

@ -104,6 +104,13 @@ impl DepotRepo {
self.cache_max_age
}
pub fn shard_dir(&self, publisher: &str) -> PathBuf {
self.root
.join("publisher")
.join(publisher)
.join("catalog2")
}
pub fn get_catalog_file_path(&self, publisher: &str, filename: &str) -> Result<PathBuf> {
let backend = self
.backend