From 5b4b719b42aae6b2d26bdd97fa624fad3dd4bb95 Mon Sep 17 00:00:00 2001 From: Till Wegmueller Date: Tue, 29 Jul 2025 19:08:18 +0200 Subject: [PATCH] Introduce cleanup functionality for obsoleted packages with TTL support - Add methods to find and clean up obsoleted packages older than a specified TTL (`find_obsoleted_packages_older_than_ttl` and `cleanup_obsoleted_packages_older_than_ttl`) in `libips`. - Implement a new `CleanupObsoleted` command in `pkg6repo` to handle cleanup operations. - Update workspace `Cargo.toml` files with unified attributes for better consistency. - Adjust dependencies (`libips` version patterns, `thiserror` updates) and enhance metadata management for obsoleted packages. - Enhance repository operations by adding batch processing and robust logging during cleanup. --- Cargo.lock | 16 ++--- crates/pkg6/Cargo.toml | 11 ++- libips/Cargo.toml | 14 ++-- libips/src/repository/obsoleted.rs | 107 +++++++++++++++++++++++++++++ pkg6depotd/Cargo.toml | 14 ++-- pkg6dev/Cargo.toml | 2 +- pkg6repo/Cargo.toml | 4 +- pkg6repo/src/main.rs | 58 ++++++++++++++++ ports/Cargo.toml | 10 ++- specfile/Cargo.toml | 10 ++- userland/Cargo.toml | 11 +-- 11 files changed, 219 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 13ad048..927b21b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -961,7 +961,7 @@ checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" [[package]] name = "libips" -version = "0.1.2" +version = "0.5.1" dependencies = [ "bincode", "chrono", @@ -1402,7 +1402,7 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "pkg6" -version = "0.1.0" +version = "0.5.1" dependencies = [ "diff-struct", "libips", @@ -1411,7 +1411,7 @@ dependencies = [ [[package]] name = "pkg6depotd" -version = "0.0.1-placeholder" +version = "0.5.1" [[package]] name = "pkg6dev" @@ -1445,7 +1445,7 @@ dependencies = [ [[package]] name = "ports" -version = "0.1.0" +version = "0.5.1" dependencies = [ "anyhow", "clap 3.2.25", @@ -1453,7 +1453,7 @@ dependencies = [ "reqwest", "shellexpand", "specfile", - "thiserror 2.0.12", + "thiserror 1.0.69", "url", "which", ] @@ -1910,12 +1910,12 @@ dependencies = [ [[package]] name = "specfile" -version = "0.1.0" +version = "0.5.1" dependencies = [ "anyhow", "pest", "pest_derive", - "thiserror 2.0.12", + "thiserror 1.0.69", ] [[package]] @@ -2319,7 +2319,7 @@ dependencies = [ [[package]] name = "userland" -version = "0.1.1" +version = "0.5.1" dependencies = [ "anyhow", "lazy_static", diff --git a/crates/pkg6/Cargo.toml b/crates/pkg6/Cargo.toml index e58f38e..449949f 100644 --- a/crates/pkg6/Cargo.toml +++ b/crates/pkg6/Cargo.toml @@ -1,9 +1,14 @@ [package] name = "pkg6" -version = "0.1.0" -edition = "2021" +version.workspace = true +authors.workspace = true +edition.workspace = true +license-file.workspace = true +repository.workspace = true +readme.workspace = true +keywords.workspace = true [dependencies] -libips = { version = "0.1.2", path = "../../libips" } +libips = { version = "*", path = "../../libips" } diff-struct = "0.5.3" serde = { version = "1.0.207", features = ["derive"] } diff --git a/libips/Cargo.toml b/libips/Cargo.toml index 855522e..6e85663 100644 --- a/libips/Cargo.toml +++ b/libips/Cargo.toml @@ -5,14 +5,14 @@ [package] name = "libips" -version = "0.1.2" -authors = ["Till Wegmueller "] -edition = "2018" -license-file = "../LICENSE" description = "The core library for the rust version of the Image Packaging System. Includes Python bindings." -repository = "https://github.com/OpenFlowLabs/libips" -readme = "../README.md" -keywords = ["packaging", "illumos"] +version.workspace = true +authors.workspace = true +edition.workspace = true +license-file.workspace = true +repository.workspace = true +readme.workspace = true +keywords.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] diff --git a/libips/src/repository/obsoleted.rs b/libips/src/repository/obsoleted.rs index ff637d8..0fbc8c9 100644 --- a/libips/src/repository/obsoleted.rs +++ b/libips/src/repository/obsoleted.rs @@ -1,6 +1,7 @@ use crate::fmri::Fmri; use crate::repository::{Result, RepositoryError}; use bincode::{deserialize, serialize}; +use chrono::{DateTime, Duration as ChronoDuration, Utc}; use miette::Diagnostic; use regex::Regex; use redb::{Database, ReadableTable, TableDefinition}; @@ -2209,6 +2210,112 @@ impl ObsoletedPackageManager { Ok(imported_count) } + /// Find obsoleted packages that are older than a specified TTL (time-to-live) + /// + /// This method finds obsoleted packages for a publisher that were obsoleted + /// more than the specified TTL duration ago. + /// + /// # Arguments + /// + /// * `publisher` - The publisher to check + /// * `ttl_days` - The TTL in days + /// + /// # Returns + /// + /// A list of FMRIs for packages that are older than the TTL + pub fn find_obsoleted_packages_older_than_ttl( + &self, + publisher: &str, + ttl_days: u32, + ) -> Result> { + // Get all obsoleted packages for the publisher + let all_packages = self.list_obsoleted_packages(publisher)?; + + // Calculate the cutoff time (current time minus TTL) + let now = Utc::now(); + let ttl_duration = ChronoDuration::days(ttl_days as i64); + let cutoff_time = now - ttl_duration; + + let mut older_packages = Vec::new(); + + // Check each package's obsolescence_date + for fmri in all_packages { + // Get the metadata for the package + if let Ok(Some(metadata)) = self.get_obsoleted_package_metadata(publisher, &fmri) { + // Parse the obsolescence_date + if let Ok(obsolescence_date) = DateTime::parse_from_rfc3339(&metadata.obsolescence_date) { + // Convert to UTC for comparison + let obsolescence_date_utc = obsolescence_date.with_timezone(&Utc); + + // Check if the package is older than the TTL + if obsolescence_date_utc < cutoff_time { + older_packages.push(fmri); + } + } else { + // If we can't parse the date, log a warning and skip this package + warn!("Failed to parse obsolescence_date for package {}: {}", + fmri, metadata.obsolescence_date); + } + } + } + + Ok(older_packages) + } + + /// Clean up obsoleted packages that are older than a specified TTL (time-to-live) + /// + /// This method finds and removes obsoleted packages for a publisher that were + /// obsoleted more than the specified TTL duration ago. + /// + /// # Arguments + /// + /// * `publisher` - The publisher to clean up + /// * `ttl_days` - The TTL in days + /// * `dry_run` - If true, only report what would be removed without actually removing + /// + /// # Returns + /// + /// The number of packages that were removed (or would be removed in dry run mode) + pub fn cleanup_obsoleted_packages_older_than_ttl( + &self, + publisher: &str, + ttl_days: u32, + dry_run: bool, + ) -> Result { + // Find packages older than the TTL + let older_packages = self.find_obsoleted_packages_older_than_ttl(publisher, ttl_days)?; + + if older_packages.is_empty() { + info!("No obsoleted packages older than {} days found for publisher {}", + ttl_days, publisher); + return Ok(0); + } + + info!("Found {} obsoleted packages older than {} days for publisher {}", + older_packages.len(), ttl_days, publisher); + + if dry_run { + // In dry run mode, just report what would be removed + for fmri in &older_packages { + info!("Would remove obsoleted package: {}", fmri); + } + return Ok(older_packages.len()); + } + + // Process packages in batches + let results = self.batch_process(publisher, &older_packages, None, |pub_name, fmri| { + info!("Removing obsoleted package: {}", fmri); + self.remove_obsoleted_package(pub_name, fmri) + })?; + + // Count successful removals + let removed_count = results.iter().filter(|r| r.as_ref().map_or(false, |&b| b)).count(); + + info!("Successfully removed {} obsoleted packages", removed_count); + + Ok(removed_count) + } + /// Batch process multiple obsoleted packages /// /// This method applies a function to multiple obsoleted packages in batch. diff --git a/pkg6depotd/Cargo.toml b/pkg6depotd/Cargo.toml index 3d76d3c..fdf544b 100644 --- a/pkg6depotd/Cargo.toml +++ b/pkg6depotd/Cargo.toml @@ -1,13 +1,13 @@ [package] name = "pkg6depotd" -version = "0.0.1-placeholder" -authors = ["Till Wegmueller "] -edition = "2018" -license-file = "LICENSE" description = "The repository server for IPS written in rust" -repository = "https://github.com/OpenFlowLabs/pkg6depotd" -readme = "README.md" -keywords = ["packaging", "illumos"] +version.workspace = true +authors.workspace = true +edition.workspace = true +license-file.workspace = true +repository.workspace = true +readme.workspace = true +keywords.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/pkg6dev/Cargo.toml b/pkg6dev/Cargo.toml index 4c6fad7..739e73e 100644 --- a/pkg6dev/Cargo.toml +++ b/pkg6dev/Cargo.toml @@ -12,7 +12,7 @@ keywords.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -libips = {path = "../libips", version = "0.1"} +libips = {path = "../libips", version = "*"} userland = {path = "../userland", version = "*"} clap = {version = "4", features = [ "derive" ] } tracing = "0.1" diff --git a/pkg6repo/Cargo.toml b/pkg6repo/Cargo.toml index 6134aa7..98177f7 100644 --- a/pkg6repo/Cargo.toml +++ b/pkg6repo/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "pkg6repo" +description = "The repository management utility for IPS written in rust" version.workspace = true authors.workspace = true edition.workspace = true license-file.workspace = true -description = "The repository management utility for IPS written in rust" repository.workspace = true readme.workspace = true keywords.workspace = true @@ -17,7 +17,7 @@ miette = { version = "7", features = ["fancy"] } thiserror = "2" tracing = "0.1" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -libips = { path = "../libips" } +libips = { path = "../libips", version = "*"} serde = { version = "1", features = ["derive"] } serde_json = "1" tempfile = "3.8" diff --git a/pkg6repo/src/main.rs b/pkg6repo/src/main.rs index 16bb22b..5a25e62 100644 --- a/pkg6repo/src/main.rs +++ b/pkg6repo/src/main.rs @@ -499,6 +499,25 @@ enum Commands { #[clap(short = 'p')] publisher: Option, }, + + /// Clean up obsoleted packages older than a specified TTL (time-to-live) + CleanupObsoleted { + /// Path or URI of the repository + #[clap(short = 's')] + repo_uri_or_path: String, + + /// Publisher to clean up obsoleted packages for + #[clap(short = 'p')] + publisher: String, + + /// TTL in days + #[clap(short = 't', long = "ttl-days", default_value = "90")] + ttl_days: u32, + + /// Perform a dry run (don't actually remove packages) + #[clap(short = 'n', long = "dry-run")] + dry_run: bool, + }, } fn main() -> Result<()> { @@ -1754,6 +1773,45 @@ fn main() -> Result<()> { }; // obsoleted_manager is dropped here, releasing the mutable borrow on repo info!("Imported {} obsoleted packages", count); + Ok(()) + }, + + Commands::CleanupObsoleted { + repo_uri_or_path, + publisher, + ttl_days, + dry_run, + } => { + if *dry_run { + info!("Dry run: Cleaning up obsoleted packages older than {} days for publisher: {}", + ttl_days, publisher); + } else { + info!("Cleaning up obsoleted packages older than {} days for publisher: {}", + ttl_days, publisher); + } + + // Open the repository + let mut repo = FileBackend::open(repo_uri_or_path)?; + + // Clean up the obsoleted packages + let count = { + // Get the obsoleted package manager + let obsoleted_manager = repo.get_obsoleted_manager()?; + + // Clean up the obsoleted packages + obsoleted_manager.cleanup_obsoleted_packages_older_than_ttl( + publisher, + *ttl_days, + *dry_run, + )? + }; // obsoleted_manager is dropped here, releasing the mutable borrow on repo + + if *dry_run { + info!("Dry run: Would remove {} obsoleted packages", count); + } else { + info!("Successfully removed {} obsoleted packages", count); + } + Ok(()) } } diff --git a/ports/Cargo.toml b/ports/Cargo.toml index 56b42c8..a3aa119 100644 --- a/ports/Cargo.toml +++ b/ports/Cargo.toml @@ -1,8 +1,12 @@ [package] name = "ports" -version = "0.1.0" -authors = ["Till Wegmueller "] -edition = "2021" +version.workspace = true +authors.workspace = true +edition.workspace = true +license-file.workspace = true +repository.workspace = true +readme.workspace = true +keywords.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/specfile/Cargo.toml b/specfile/Cargo.toml index 9bc27b8..3f3aa3b 100644 --- a/specfile/Cargo.toml +++ b/specfile/Cargo.toml @@ -1,8 +1,12 @@ [package] name = "specfile" -version = "0.1.0" -authors = ["Till Wegmueller "] -edition = "2021" +version.workspace = true +authors.workspace = true +edition.workspace = true +license-file.workspace = true +repository.workspace = true +readme.workspace = true +keywords.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/userland/Cargo.toml b/userland/Cargo.toml index 443bb70..b2fb26f 100644 --- a/userland/Cargo.toml +++ b/userland/Cargo.toml @@ -1,10 +1,13 @@ [package] name = "userland" -version = "0.1.1" -authors = ["Till Wegmueller "] -edition = "2021" -license-file = "LICENSE" description = "Helper tool for IPS package development" +version.workspace = true +authors.workspace = true +edition.workspace = true +license-file.workspace = true +repository.workspace = true +readme.workspace = true +keywords.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html