diff --git a/Cargo.lock b/Cargo.lock index 6842ae4..a57d3f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -973,6 +973,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tracing", + "walkdir", ] [[package]] @@ -1691,6 +1692,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.27" @@ -2336,6 +2346,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" diff --git a/libips/Cargo.toml b/libips/Cargo.toml index 847e3a9..0ffac0d 100644 --- a/libips/Cargo.toml +++ b/libips/Cargo.toml @@ -35,3 +35,4 @@ semver = { version = "1.0.20", features = ["serde"] } diff-struct = "0.5.3" chrono = "0.4.41" tempfile = "3.20.0" +walkdir = "2.4.0" diff --git a/libips/src/repository/file_backend.rs b/libips/src/repository/file_backend.rs index 9126a80..52891da 100644 --- a/libips/src/repository/file_backend.rs +++ b/libips/src/repository/file_backend.rs @@ -18,6 +18,7 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use std::time::{SystemTime, UNIX_EPOCH}; use tracing::{debug, error, info}; +use walkdir::WalkDir; use crate::actions::{File as FileAction, Manifest}; use crate::digest::Digest; @@ -1988,318 +1989,120 @@ impl FileBackend { // Check if the publisher directory exists if publisher_pkg_dir.exists() { - // Walk through the directory and process package manifests - if let Ok(entries) = fs::read_dir(&publisher_pkg_dir) { - for entry in entries.flatten() { - let path = entry.path(); - - if path.is_dir() { - // Recursively search subdirectories - if let Ok(subentries) = fs::read_dir(&path) { - for subentry in subentries.flatten() { - let subpath = subentry.path(); - if subpath.is_file() { - // Try to read the first few bytes of the file to check if it's a manifest file - let mut file = match fs::File::open(&subpath) { - Ok(file) => file, - Err(err) => { - error!( - "FileBackend::build_search_index: Error opening file {}: {}", - subpath.display(), - err - ); - continue; - } - }; - - let mut buffer = [0; 1024]; - let bytes_read = match file.read(&mut buffer) { - Ok(bytes) => bytes, - Err(err) => { - error!( - "FileBackend::build_search_index: Error reading file {}: {}", - subpath.display(), - err - ); - continue; - } - }; - - // Check if the file starts with a valid manifest marker - if bytes_read == 0 - || (buffer[0] != b'{' && buffer[0] != b'<' && buffer[0] != b's') - { - continue; - } - - // Parse the manifest file to get package information - match Manifest::parse_file(&subpath) { - Ok(manifest) => { - // Look for the pkg.fmri attribute - for attr in &manifest.attributes { - if attr.key == "pkg.fmri" && !attr.values.is_empty() { - let fmri_str = &attr.values[0]; - - // Parse the FMRI using our Fmri type - match Fmri::parse(fmri_str) { - Ok(parsed_fmri) => { - // Create a PackageInfo struct - let package_info = PackageInfo { - fmri: parsed_fmri.clone(), - }; - - // Create a PackageContents struct - let version = parsed_fmri.version(); - let package_id = if !version.is_empty() { - format!("{}@{}", parsed_fmri.stem(), version) - } else { - parsed_fmri.stem().to_string() - }; - - // Extract content information - let files = if !manifest.files.is_empty() { - Some( - manifest - .files - .iter() - .map(|f| f.path.clone()) - .collect(), - ) - } else { - None - }; - - let directories = - if !manifest.directories.is_empty() { - Some( - manifest - .directories - .iter() - .map(|d| d.path.clone()) - .collect(), - ) - } else { - None - }; - - let links = if !manifest.links.is_empty() { - Some( - manifest - .links - .iter() - .map(|l| l.path.clone()) - .collect(), - ) - } else { - None - }; - - let dependencies = - if !manifest.dependencies.is_empty() { - Some( - manifest - .dependencies - .iter() - .filter_map(|d| { - d.fmri - .as_ref() - .map(|f| f.to_string()) - }) - .collect(), - ) - } else { - None - }; - - let licenses = if !manifest.licenses.is_empty() { - Some( - manifest - .licenses - .iter() - .map(|l| { - if let Some(path_prop) = - l.properties.get("path") - { - path_prop.value.clone() - } else if let Some(license_prop) = - l.properties.get("license") - { - license_prop.value.clone() - } else { - l.payload.clone() - } - }) - .collect(), - ) - } else { - None - }; - - // Create a PackageContents struct - let package_contents = PackageContents { - package_id, - files, - directories, - links, - dependencies, - licenses, - }; - - // Add the package to the index - index.add_package(&package_info, Some(&package_contents)); - } - Err(err) => { - // Log the error but continue processing - error!( - "FileBackend::build_search_index: Error parsing FMRI '{}': {}", - fmri_str, err - ); - } - } - } - } - } - Err(err) => { - // Log the error but continue processing other files - error!( - "FileBackend::build_search_index: Error parsing manifest file {}: {}", - subpath.display(), - err - ); - } - } - } - } - } - } else if path.is_file() { - // Try to read the first few bytes of the file to check if it's a manifest file - let mut file = match fs::File::open(&path) { - Ok(file) => file, - Err(err) => { - error!( - "FileBackend::build_search_index: Error opening file {}: {}", - path.display(), - err - ); - continue; - } - }; - - let mut buffer = [0; 1024]; - let bytes_read = match file.read(&mut buffer) { - Ok(bytes) => bytes, - Err(err) => { - error!( - "FileBackend::build_search_index: Error reading file {}: {}", - path.display(), - err - ); - continue; - } - }; - - // Check if the file starts with a valid manifest marker - if bytes_read == 0 - || (buffer[0] != b'{' && buffer[0] != b'<' && buffer[0] != b's') - { + // Use walkdir to recursively walk through the directory and process package manifests + for entry in WalkDir::new(&publisher_pkg_dir) + .follow_links(true) + .into_iter() + .filter_map(|e| e.ok()) + { + let path = entry.path(); + + if path.is_file() { + // Try to read the first few bytes of the file to check if it's a manifest file + let mut file = match fs::File::open(&path) { + Ok(file) => file, + Err(err) => { + error!( + "FileBackend::build_search_index: Error opening file {}: {}", + path.display(), + err + ); continue; } - // Parse the manifest file to get package information - match Manifest::parse_file(&path) { - Ok(manifest) => { - // Look for the pkg.fmri attribute - for attr in &manifest.attributes { - if attr.key == "pkg.fmri" && !attr.values.is_empty() { - let fmri_str = &attr.values[0]; + }; - // Parse the FMRI using our Fmri type - match Fmri::parse(fmri_str) { - Ok(parsed_fmri) => { - // Create a PackageInfo struct - let package_info = PackageInfo { - fmri: parsed_fmri.clone(), - }; + let mut buffer = [0; 1024]; + let bytes_read = match file.read(&mut buffer) { + Ok(bytes) => bytes, + Err(err) => { + error!( + "FileBackend::build_search_index: Error reading file {}: {}", + path.display(), + err + ); + continue; + } + }; - // Create a PackageContents struct - let version = parsed_fmri.version(); - let package_id = if !version.is_empty() { - format!("{}@{}", parsed_fmri.stem(), version) - } else { - parsed_fmri.stem().to_string() - }; + // Check if the file starts with a valid manifest marker + if bytes_read == 0 + || (buffer[0] != b'{' && buffer[0] != b'<' && buffer[0] != b's') + { + continue; + } - // Extract content information - let files = if !manifest.files.is_empty() { + // Parse the manifest file to get package information + match Manifest::parse_file(&path) { + Ok(manifest) => { + // Look for the pkg.fmri attribute + for attr in &manifest.attributes { + if attr.key == "pkg.fmri" && !attr.values.is_empty() { + let fmri_str = &attr.values[0]; + + // Parse the FMRI using our Fmri type + match Fmri::parse(fmri_str) { + Ok(parsed_fmri) => { + // Create a PackageInfo struct + let package_info = PackageInfo { + fmri: parsed_fmri.clone(), + }; + + // Create a PackageContents struct + let version = parsed_fmri.version(); + let package_id = if !version.is_empty() { + format!("{}@{}", parsed_fmri.stem(), version) + } else { + parsed_fmri.stem().to_string() + }; + + // Extract content information + let files = if !manifest.files.is_empty() { + Some( + manifest + .files + .iter() + .map(|f| f.path.clone()) + .collect(), + ) + } else { + None + }; + + let directories = + if !manifest.directories.is_empty() { Some( manifest - .files + .directories .iter() - .map(|f| f.path.clone()) + .map(|d| d.path.clone()) .collect(), ) } else { None }; - let directories = - if !manifest.directories.is_empty() { - Some( - manifest - .directories - .iter() - .map(|d| d.path.clone()) - .collect(), - ) - } else { - None - }; + let links = if !manifest.links.is_empty() { + Some( + manifest + .links + .iter() + .map(|l| l.path.clone()) + .collect(), + ) + } else { + None + }; - let links = if !manifest.links.is_empty() { + let dependencies = + if !manifest.dependencies.is_empty() { Some( manifest - .links + .dependencies .iter() - .map(|l| l.path.clone()) - .collect(), - ) - } else { - None - }; - - let dependencies = - if !manifest.dependencies.is_empty() { - Some( - manifest - .dependencies - .iter() - .filter_map(|d| { - d.fmri - .as_ref() - .map(|f| f.to_string()) - }) - .collect(), - ) - } else { - None - }; - - let licenses = if !manifest.licenses.is_empty() { - Some( - manifest - .licenses - .iter() - .map(|l| { - if let Some(path_prop) = - l.properties.get("path") - { - path_prop.value.clone() - } else if let Some(license_prop) = - l.properties.get("license") - { - license_prop.value.clone() - } else { - l.payload.clone() - } + .filter_map(|d| { + d.fmri + .as_ref() + .map(|f| f.to_string()) }) .collect(), ) @@ -2307,43 +2110,64 @@ impl FileBackend { None }; - let package_contents = PackageContents { - package_id, - files, - directories, - links, - dependencies, - licenses, - }; + let licenses = if !manifest.licenses.is_empty() { + Some( + manifest + .licenses + .iter() + .map(|l| { + if let Some(path_prop) = + l.properties.get("path") + { + path_prop.value.clone() + } else if let Some(license_prop) = + l.properties.get("license") + { + license_prop.value.clone() + } else { + l.payload.clone() + } + }) + .collect(), + ) + } else { + None + }; - // Add the package to the index - index.add_package( - &package_info, - Some(&package_contents), - ); + // Create a PackageContents struct + let package_contents = PackageContents { + package_id, + files, + directories, + links, + dependencies, + licenses, + }; - // Found the package info, no need to check other attributes - break; - } - Err(err) => { - // Log the error but continue processing - error!( + // Add the package to the index + index.add_package(&package_info, Some(&package_contents)); + + // Found the package info, no need to check other attributes + break; + } + Err(err) => { + // Log the error but continue processing + error!( "FileBackend::build_search_index: Error parsing FMRI '{}': {}", - fmri_str, err - ); - } + fmri_str, err + ); } } } } - Err(err) => { - // Log the error but continue processing other files - error!( + } + Err(err) => { + // Log the error but continue processing other files + error!( "FileBackend::build_search_index: Error parsing manifest file {}: {}", - path.display(), - err - ); - } + path.display(), + err + ); } } }