From d483e2a9952e64c32829cca248cbe1c623d6ad10 Mon Sep 17 00:00:00 2001 From: Till Wegmueller Date: Wed, 13 Aug 2025 23:23:45 +0200 Subject: [PATCH] implement manifest fetching and fix CI errors --- libips/src/fmri.rs | 4 +- libips/src/repository/file_backend.rs | 109 +++++++++++++++++++++ libips/src/repository/mod.rs | 19 ++++ libips/src/repository/rest_backend.rs | 133 +++++++++++++++++++++++++- libips/src/solver/mod.rs | 7 +- 5 files changed, 267 insertions(+), 5 deletions(-) diff --git a/libips/src/fmri.rs b/libips/src/fmri.rs index 6adefcc..1cfde9c 100644 --- a/libips/src/fmri.rs +++ b/libips/src/fmri.rs @@ -378,7 +378,7 @@ impl Version { /// Parse a version string into a Version /// - /// The version string should be in the format: release[,branch][-build][:timestamp] + /// The version string should be in the format: release\[,branch\]\[-build\]\[:timestamp\] pub fn parse(version_str: &str) -> Result { let mut version = Version { release: String::new(), @@ -574,7 +574,7 @@ impl Fmri { /// Parse an FMRI string into an Fmri /// - /// The FMRI string should be in the format: [scheme://][publisher/]name[@version] + /// The FMRI string should be in the format: \[scheme://\]\[publisher/\]name[@version] pub fn parse(fmri_str: &str) -> Result { let mut fmri = Fmri { scheme: "pkg".to_string(), diff --git a/libips/src/repository/file_backend.rs b/libips/src/repository/file_backend.rs index 954e3eb..b692f97 100644 --- a/libips/src/repository/file_backend.rs +++ b/libips/src/repository/file_backend.rs @@ -1240,6 +1240,115 @@ impl ReadableRepository for FileBackend { Ok(package_contents) } + fn fetch_payload(&mut self, publisher: &str, digest: &str, dest: &Path) -> Result<()> { + // Parse digest; supports both raw hash and source:algorithm:hash + let parsed = match Digest::from_str(digest) { + Ok(d) => d, + Err(e) => return Err(RepositoryError::DigestError(e.to_string())), + }; + let hash = parsed.hash.clone(); + let algo = parsed.algorithm.clone(); + + if hash.is_empty() { + return Err(RepositoryError::Other("Empty digest provided".to_string())); + } + + // Prepare candidate paths (prefer publisher-specific, then global) + let cand_pub = Self::construct_file_path_with_publisher(&self.path, publisher, &hash); + let cand_global = Self::construct_file_path(&self.path, &hash); + + let source_path = if cand_pub.exists() { + cand_pub + } else if cand_global.exists() { + cand_global + } else { + return Err(RepositoryError::NotFound(format!( + "payload {} not found in repository", + hash + ))); + }; + + // Ensure destination directory exists + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent)?; + } + + // If destination already exists and matches digest, do nothing + if dest.exists() { + let bytes = fs::read(dest)?; + match crate::digest::Digest::from_bytes(&bytes, algo.clone(), crate::digest::DigestSource::PrimaryPayloadHash) { + Ok(comp) if comp.hash == hash => return Ok(()), + _ => { /* fall through to overwrite */ } + } + } + + // Read source content and verify digest + let bytes = fs::read(&source_path)?; + match crate::digest::Digest::from_bytes(&bytes, algo, crate::digest::DigestSource::PrimaryPayloadHash) { + Ok(comp) => { + if comp.hash != hash { + return Err(RepositoryError::DigestError(format!( + "Digest mismatch: expected {}, got {}", + hash, comp.hash + ))); + } + } + Err(e) => return Err(RepositoryError::DigestError(e.to_string())), + } + + // Write atomically + let tmp = dest.with_extension("tmp"); + { + let mut f = File::create(&tmp)?; + f.write_all(&bytes)?; + } + fs::rename(&tmp, dest)?; + + Ok(()) + } + + fn fetch_manifest( + &mut self, + publisher: &str, + fmri: &crate::fmri::Fmri, + ) -> Result { + // Require a concrete version + let version = fmri.version(); + if version.is_empty() { + return Err(RepositoryError::Other("FMRI must include a version to fetch manifest".into())); + } + + // Preferred path: publisher-scoped manifest path + let path = Self::construct_manifest_path(&self.path, publisher, fmri.stem(), &version); + if path.exists() { + return crate::actions::Manifest::parse_file(&path).map_err(RepositoryError::from); + } + + // Fallbacks: global pkg layout without publisher + let encoded_stem = Self::url_encode(fmri.stem()); + let encoded_version = Self::url_encode(&version); + let alt1 = self.path.join("pkg").join(&encoded_stem).join(&encoded_version); + if alt1.exists() { + return crate::actions::Manifest::parse_file(&alt1).map_err(RepositoryError::from); + } + + let alt2 = self + .path + .join("publisher") + .join(publisher) + .join("pkg") + .join(&encoded_stem) + .join(&encoded_version); + if alt2.exists() { + return crate::actions::Manifest::parse_file(&alt2).map_err(RepositoryError::from); + } + + Err(RepositoryError::NotFound(format!( + "manifest for {} not found", + fmri + ))) + } + /// Search for packages in the repository fn search( &self, diff --git a/libips/src/repository/mod.rs b/libips/src/repository/mod.rs index 373e10c..e708042 100644 --- a/libips/src/repository/mod.rs +++ b/libips/src/repository/mod.rs @@ -331,6 +331,25 @@ pub trait ReadableRepository { action_types: Option<&[String]>, ) -> Result>; + /// Fetch a content payload identified by digest into the destination path. + /// Implementations should download/copy the payload to a temporary path, + /// verify integrity, and atomically move into `dest`. + fn fetch_payload( + &mut self, + publisher: &str, + digest: &str, + dest: &Path, + ) -> Result<()>; + + /// Fetch a package manifest by FMRI from the repository. + /// Implementations should retrieve and parse the manifest for the given + /// publisher and fully-qualified FMRI (name@version). + fn fetch_manifest( + &mut self, + publisher: &str, + fmri: &crate::fmri::Fmri, + ) -> Result; + /// Search for packages in the repository /// /// This method searches for packages in the repository using the search index. diff --git a/libips/src/repository/rest_backend.rs b/libips/src/repository/rest_backend.rs index cc65810..c6b388e 100644 --- a/libips/src/repository/rest_backend.rs +++ b/libips/src/repository/rest_backend.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use std::fs::{self, File}; use std::io::Write; use std::path::{Path, PathBuf}; +use std::str::FromStr; use tracing::{debug, info, warn}; use reqwest::blocking::Client; @@ -33,7 +34,7 @@ use super::catalog::CatalogManager; /// use std::path::Path; /// /// // Open a connection to a remote repository -/// let mut repo = RestBackend::open("http://pkg.opensolaris.org/release").unwrap(); +/// let mut repo = RestBackend::open("https://pkg.opensolaris.org/release").unwrap(); /// /// // Set a local cache path for downloaded catalog files /// repo.set_local_cache_path(Path::new("/tmp/pkg_cache")).unwrap(); @@ -535,6 +536,136 @@ impl ReadableRepository for RestBackend { Ok(package_contents) } + fn fetch_payload( + &mut self, + publisher: &str, + digest: &str, + dest: &Path, + ) -> Result<()> { + // Determine hash and algorithm from the provided digest string + let mut hash = digest.to_string(); + let mut algo: Option = None; + if digest.contains(':') { + if let Ok(d) = crate::digest::Digest::from_str(digest) { + hash = d.hash.clone(); + algo = Some(d.algorithm); + } + } + + if hash.is_empty() { + return Err(RepositoryError::Other("Empty digest provided".to_string())); + } + + let shard = if hash.len() >= 2 { &hash[0..2] } else { &hash[..] }; + let candidates = vec![ + format!("{}/file/{}/{}", self.uri, shard, hash), + format!("{}/publisher/{}/file/{}/{}", self.uri, publisher, shard, hash), + ]; + + // Ensure destination directory exists + if let Some(parent) = dest.parent() { + fs::create_dir_all(parent)?; + } + + let mut last_err: Option = None; + for url in candidates { + match self.client.get(&url).send() { + Ok(resp) if resp.status().is_success() => { + let body = resp.bytes().map_err(|e| RepositoryError::Other(format!("Failed to read payload body: {}", e)))?; + + // Verify digest if algorithm is known + if let Some(alg) = algo.clone() { + match crate::digest::Digest::from_bytes(&body, alg, crate::digest::DigestSource::PrimaryPayloadHash) { + Ok(comp) => { + if comp.hash != hash { + return Err(RepositoryError::DigestError(format!( + "Digest mismatch: expected {}, got {}", + hash, comp.hash + ))); + } + } + Err(e) => return Err(RepositoryError::DigestError(format!("{}", e))), + } + } + + // Write atomically + let tmp = dest.with_extension("tmp"); + let mut f = File::create(&tmp)?; + f.write_all(&body)?; + drop(f); + fs::rename(&tmp, dest)?; + return Ok(()); + } + Ok(resp) => { + last_err = Some(format!("HTTP {} for {}", resp.status(), url)); + } + Err(e) => { + last_err = Some(format!("{} for {}", e, url)); + } + } + } + + Err(RepositoryError::NotFound(last_err.unwrap_or_else(|| "payload not found".to_string()))) + } + + fn fetch_manifest( + &mut self, + publisher: &str, + fmri: &crate::fmri::Fmri, + ) -> Result { + // Require versioned FMRI + let version = fmri.version(); + if version.is_empty() { + return Err(RepositoryError::Other("FMRI must include a version to fetch manifest".into())); + } + + // URL-encode helper + let url_encode = |s: &str| -> String { + let mut out = String::new(); + for b in s.bytes() { + match b { + b'-' | b'_' | b'.' | b'~' | b'0'..=b'9' | b'a'..=b'z' | b'A'..=b'Z' => out.push(b as char), + b' ' => out.push('+'), + _ => { + out.push('%'); + out.push_str(&format!("{:02X}", b)); + } + } + } + out + }; + + let encoded_fmri = url_encode(&format!("{}@{}", fmri.stem(), version)); + let encoded_stem = url_encode(fmri.stem()); + let encoded_version = url_encode(&version); + + let candidates = vec![ + format!("{}/manifest/0/{}", self.uri, encoded_fmri), + format!("{}/publisher/{}/manifest/0/{}", self.uri, publisher, encoded_fmri), + // Fallbacks to direct file-style paths if server exposes static files + format!("{}/pkg/{}/{}", self.uri, encoded_stem, encoded_version), + format!("{}/publisher/{}/pkg/{}/{}", self.uri, publisher, encoded_stem, encoded_version), + ]; + + let mut last_err: Option = None; + for url in candidates { + match self.client.get(&url).send() { + Ok(resp) if resp.status().is_success() => { + let text = resp.text().map_err(|e| RepositoryError::Other(format!("Failed to read manifest body: {}", e)))?; + return crate::actions::Manifest::parse_string(text).map_err(RepositoryError::from); + } + Ok(resp) => { + last_err = Some(format!("HTTP {} for {}", resp.status(), url)); + } + Err(e) => { + last_err = Some(format!("{} for {}", e, url)); + } + } + } + + Err(RepositoryError::NotFound(last_err.unwrap_or_else(|| "manifest not found".to_string()))) + } + fn search( &self, _query: &str, diff --git a/libips/src/solver/mod.rs b/libips/src/solver/mod.rs index 1de29d7..7034b54 100644 --- a/libips/src/solver/mod.rs +++ b/libips/src/solver/mod.rs @@ -25,10 +25,11 @@ use resolvo::{self, Candidates, Dependencies as RDependencies, DependencyProvide use miette::Diagnostic; use thiserror::Error; -use crate::actions::{Dependency, Manifest}; +use crate::actions::Manifest; #[derive(Clone, Debug)] struct PkgCand { + #[allow(dead_code)] id: SolvableId, name_id: NameId, fmri: Fmri, @@ -58,7 +59,7 @@ struct IpsProvider<'a> { // per-name publisher preference order; set by dependency processing or top-level specs publisher_prefs: RefCell>>, } -use crate::fmri::{Fmri, Version}; +use crate::fmri::Fmri; use crate::image::{catalog::PackageInfo, Image}; impl<'a> IpsProvider<'a> { @@ -577,6 +578,8 @@ mod solver_integration_tests { use crate::image::catalog::{CATALOG_TABLE, OBSOLETED_TABLE}; use redb::Database; use tempfile::tempdir; + use crate::fmri::Version; + use crate::actions::Dependency; fn mk_version(release: &str, branch: Option<&str>, timestamp: Option<&str>) -> Version { let mut v = Version::new(release);