Replace legacy setup_test_env.sh script with xtask for environment setup, refactor e2e tests to use pre-built binaries, and update xtask commands and documentation for improved automation and reliability.

pkg6repo/Cargo.toml

@@ -18,3 +18,7 @@ thiserror = "1.0.50"
 libips = { path = "../libips" }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
+
+[[test]]
+name = "e2e_tests"
+path = "src/e2e_tests.rs"

pkg6repo/src/e2e_tests.rs

@@ -5,6 +5,7 @@
 
 #[cfg(test)]
 mod e2e_tests {
+    use std::env;
     use std::fs;
     use std::path::PathBuf;
     use std::process::Command;
@@ -12,6 +13,17 @@ mod e2e_tests {
 
     // The base directory for all test repositories
     const TEST_REPO_BASE_DIR: &str = "/tmp/pkg6repo_e2e_test";
+    
+    // Get the path to the pre-built binaries
+    fn get_bin_dir() -> PathBuf {
+        match env::var("PKG6_TEST_BIN_DIR") {
+            Ok(dir) => PathBuf::from(dir),
+            Err(_) => {
+                // Fallback to the default location if the environment variable is not set
+                PathBuf::from("/tmp/pkg6_test/bin")
+            }
+        }
+    }
 
     // Helper function to create a unique test directory
     fn create_test_dir(test_name: &str) -> PathBuf {
@@ -35,24 +47,20 @@ mod e2e_tests {
         }
     }
 
-    // Helper function to run the setup script
+    // Helper function to set up the test environment
     fn run_setup_script() -> (PathBuf, PathBuf) {
-        // Get the project root directory
+        // Run the xtask setup-test-env command
-        let output = Command::new("git")
+        let output = Command::new("cargo")
-            .args(["rev-parse", "--show-toplevel"])
+            .args(["run", "-p", "xtask", "--", "setup-test-env"])
             .output()
-            .expect("Failed to execute git command");
+            .expect("Failed to run xtask setup-test-env");
-
+            
-        let project_root = String::from_utf8(output.stdout)
+        if !output.status.success() {
-            .expect("Invalid UTF-8 output")
+            panic!(
-            .trim()
+                "Failed to set up test environment: {}",
-            .to_string();
+                String::from_utf8_lossy(&output.stderr)
-
+            );
-        // Run the setup script
+        }
-        Command::new("bash")
-            .arg(format!("{}/setup_test_env.sh", project_root))
-            .status()
-            .expect("Failed to run setup script");
 
         // Return the paths to the prototype and manifest directories
         (
@@ -63,10 +71,18 @@ mod e2e_tests {
 
     // Helper function to run pkg6repo command
     fn run_pkg6repo(args: &[&str]) -> Result<String, String> {
-        let output = Command::new("cargo")
+        let bin_dir = get_bin_dir();
-            .arg("run")
+        let pkg6repo_bin = bin_dir.join("pkg6repo");
-            .arg("--bin")
+        
-            .arg("pkg6repo")
+        // Check if the binary exists
+        if !pkg6repo_bin.exists() {
+            return Err(format!(
+                "pkg6repo binary not found at {}. Run 'cargo xtask build-e2e' first.",
+                pkg6repo_bin.display()
+            ));
+        }
+        
+        let output = Command::new(pkg6repo_bin)
             .args(args)
             .output()
             .expect("Failed to execute pkg6repo command");
@@ -80,10 +96,18 @@ mod e2e_tests {
 
     // Helper function to run pkg6dev command
     fn run_pkg6dev(args: &[&str]) -> Result<String, String> {
-        let output = Command::new("cargo")
+        let bin_dir = get_bin_dir();
-            .arg("run")
+        let pkg6dev_bin = bin_dir.join("pkg6dev");
-            .arg("--bin")
+        
-            .arg("pkg6dev")
+        // Check if the binary exists
+        if !pkg6dev_bin.exists() {
+            return Err(format!(
+                "pkg6dev binary not found at {}. Run 'cargo xtask build-e2e' first.",
+                pkg6dev_bin.display()
+            ));
+        }
+        
+        let output = Command::new(pkg6dev_bin)
             .args(args)
             .output()
             .expect("Failed to execute pkg6dev command");
@@ -181,9 +205,9 @@ mod e2e_tests {
         let manifest_path = manifest_dir.join("example.p5m");
         let result = run_pkg6dev(&[
             "publish",
-            manifest_path.to_str().unwrap(),
+            "--manifest-path", manifest_path.to_str().unwrap(),
-            prototype_dir.to_str().unwrap(),
+            "--prototype-dir", prototype_dir.to_str().unwrap(),
-            repo_path.to_str().unwrap(),
+            "--repo-path", repo_path.to_str().unwrap(),
         ]);
         assert!(
             result.is_ok(),
@@ -238,9 +262,9 @@ mod e2e_tests {
         let manifest_path = manifest_dir.join("example.p5m");
         let result = run_pkg6dev(&[
             "publish",
-            manifest_path.to_str().unwrap(),
+            "--manifest-path", manifest_path.to_str().unwrap(),
-            prototype_dir.to_str().unwrap(),
+            "--prototype-dir", prototype_dir.to_str().unwrap(),
-            repo_path.to_str().unwrap(),
+            "--repo-path", repo_path.to_str().unwrap(),
         ]);
         assert!(
             result.is_ok(),
@@ -303,9 +327,9 @@ mod e2e_tests {
         let manifest_path1 = manifest_dir.join("example.p5m");
         let result = run_pkg6dev(&[
             "publish",
-            manifest_path1.to_str().unwrap(),
+            "--manifest-path", manifest_path1.to_str().unwrap(),
-            prototype_dir.to_str().unwrap(),
+            "--prototype-dir", prototype_dir.to_str().unwrap(),
-            repo_path.to_str().unwrap(),
+            "--repo-path", repo_path.to_str().unwrap(),
         ]);
         assert!(
             result.is_ok(),
@@ -317,9 +341,9 @@ mod e2e_tests {
         let manifest_path2 = manifest_dir.join("example2.p5m");
         let result = run_pkg6dev(&[
             "publish",
-            manifest_path2.to_str().unwrap(),
+            "--manifest-path", manifest_path2.to_str().unwrap(),
-            prototype_dir.to_str().unwrap(),
+            "--prototype-dir", prototype_dir.to_str().unwrap(),
-            repo_path.to_str().unwrap(),
+            "--repo-path", repo_path.to_str().unwrap(),
         ]);
         assert!(
             result.is_ok(),

setup_test_env.sh

@@ -1,74 +0,0 @@
-#!/bin/bash
-# Script to set up the test environment for repository tests
-
-set -e  # Exit on error
-
-# Directory where test files will be created
-TEST_BASE_DIR="/tmp/pkg6_test"
-PROTOTYPE_DIR="$TEST_BASE_DIR/prototype"
-MANIFEST_DIR="$TEST_BASE_DIR/manifests"
-
-# Clean up any existing test directories
-if [ -d "$TEST_BASE_DIR" ]; then
-    echo "Cleaning up existing test directory..."
-    rm -rf "$TEST_BASE_DIR"
-fi
-
-# Create test directories
-echo "Creating test directories..."
-mkdir -p "$PROTOTYPE_DIR"
-mkdir -p "$MANIFEST_DIR"
-
-# Compile the applications
-echo "Compiling applications..."
-cd "$(dirname "$0")"
-cargo build
-
-# Create a simple prototype directory structure with some files
-echo "Creating prototype directory structure..."
-
-# Create some directories
-mkdir -p "$PROTOTYPE_DIR/usr/bin"
-mkdir -p "$PROTOTYPE_DIR/usr/share/doc/example"
-mkdir -p "$PROTOTYPE_DIR/etc/config"
-
-# Create some files
-echo "#!/bin/sh\necho 'Hello, World!'" > "$PROTOTYPE_DIR/usr/bin/hello"
-chmod +x "$PROTOTYPE_DIR/usr/bin/hello"
-
-echo "This is an example document." > "$PROTOTYPE_DIR/usr/share/doc/example/README.txt"
-
-echo "# Example configuration file\nvalue=42" > "$PROTOTYPE_DIR/etc/config/example.conf"
-
-# Create a simple manifest
-echo "Creating package manifest..."
-cat > "$MANIFEST_DIR/example.p5m" << EOF
-set name=pkg.fmri value=pkg://test/example@1.0.0
-set name=pkg.summary value="Example package for testing"
-set name=pkg.description value="This is an example package used for testing the repository implementation."
-set name=info.classification value="org.opensolaris.category.2008:System/Core"
-set name=variant.arch value=i386 value=sparc
-file path=usr/bin/hello mode=0755 owner=root group=bin
-file path=usr/share/doc/example/README.txt mode=0644 owner=root group=bin
-file path=etc/config/example.conf mode=0644 owner=root group=bin preserve=true
-dir path=usr/bin mode=0755 owner=root group=bin
-dir path=usr/share/doc/example mode=0755 owner=root group=bin
-dir path=etc/config mode=0755 owner=root group=sys
-EOF
-
-# Create a second manifest for testing multiple packages
-cat > "$MANIFEST_DIR/example2.p5m" << EOF
-set name=pkg.fmri value=pkg://test/example2@1.0.0
-set name=pkg.summary value="Second example package for testing"
-set name=pkg.description value="This is a second example package used for testing the repository implementation."
-set name=info.classification value="org.opensolaris.category.2008:System/Core"
-set name=variant.arch value=i386 value=sparc
-file path=usr/bin/hello mode=0755 owner=root group=bin
-file path=usr/share/doc/example/README.txt mode=0644 owner=root group=bin
-dir path=usr/bin mode=0755 owner=root group=bin
-dir path=usr/share/doc/example mode=0755 owner=root group=bin
-EOF
-
-echo "Test environment setup complete!"
-echo "Prototype directory: $PROTOTYPE_DIR"
-echo "Manifest directory: $MANIFEST_DIR"

xtask/README.md

@@ -0,0 +1,78 @@
+# Xtask for IPS
+
+This directory contains the xtask implementation for the IPS project. Xtask is a Rust-based build system that allows us to write build scripts and automation tasks in Rust instead of shell scripts, making them more maintainable and cross-platform.
+
+## Available Commands
+
+The following commands are available through cargo-xtask:
+
+```bash
+cargo run -p xtask -- setup-test-env    # Set up the test environment for repository tests
+cargo run -p xtask -- build             # Build the project
+cargo run -p xtask -- build -r          # Build with release optimizations
+cargo run -p xtask -- build -p <crate>  # Build a specific crate
+cargo run -p xtask -- test              # Run tests
+cargo run -p xtask -- test -r           # Run tests with release optimizations
+cargo run -p xtask -- test -p <crate>   # Run tests for a specific crate
+cargo run -p xtask -- build-e2e         # Build binaries for end-to-end tests
+cargo run -p xtask -- run-e2e           # Run end-to-end tests using pre-built binaries
+cargo run -p xtask -- run-e2e -t <test> # Run a specific end-to-end test
+cargo run -p xtask -- fmt               # Format code using cargo fmt
+cargo run -p xtask -- clippy            # Run clippy for code quality checks
+cargo run -p xtask -- clean             # Clean build artifacts
+```
+
+## End-to-End Testing
+
+End-to-end tests are an important part of the IPS project. They test the entire system from the user's perspective, ensuring that all components work together correctly.
+
+### Improved End-to-End Testing Approach
+
+To reduce flaky tests and improve reliability, we've separated the building of binaries from the test execution. This approach has several advantages:
+
+1. **Reduced Flakiness**: By separating the build step from the test execution, we reduce the chance of tests failing due to build issues.
+2. **Faster Test Execution**: Pre-building the binaries means that tests can start immediately without waiting for compilation.
+3. **Consistent Test Environment**: Using xtask for both building binaries and setting up the test environment ensures consistency.
+
+### How to Run End-to-End Tests
+
+To run end-to-end tests, follow these steps:
+
+1. Build the binaries for end-to-end tests:
+   ```bash
+   cargo run -p xtask -- build-e2e
+   ```
+
+2. Run the end-to-end tests:
+   ```bash
+   cargo run -p xtask -- run-e2e
+   ```
+
+   To run a specific test:
+   ```bash
+   cargo run -p xtask -- run-e2e -t test_e2e_create_repository
+   ```
+
+### How It Works
+
+The `build-e2e` command:
+- Builds the necessary binaries (pkg6repo, pkg6dev, etc.) in release mode
+- Copies the binaries to a dedicated directory (`/tmp/pkg6_test/bin`)
+
+The `run-e2e` command:
+- Checks if the pre-built binaries exist, and builds them if they don't
+- Sets up the test environment using the `setup-test-env` command
+- Runs the end-to-end tests, passing the location of the pre-built binaries via an environment variable
+
+The end-to-end tests:
+- Use the pre-built binaries instead of compiling them on the fly
+- Use a consistent test environment setup
+
+## Adding New Commands
+
+To add a new command to cargo-xtask:
+
+1. Edit the `xtask/src/main.rs` file
+2. Add a new variant to the `Commands` enum
+3. Implement a function for the new command
+4. Add a match arm in the `main` function to call your new function

xtask/src/main.rs

@@ -2,13 +2,14 @@ use anyhow::{Context, Result};
 use clap::{Parser, Subcommand};
 use std::fs::{self, File};
 use std::io::Write;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 
 // Constants
 const TEST_BASE_DIR: &str = "/tmp/pkg6_test";
 const PROTOTYPE_DIR: &str = "/tmp/pkg6_test/prototype";
 const MANIFEST_DIR: &str = "/tmp/pkg6_test/manifests";
+const E2E_TEST_BIN_DIR: &str = "/tmp/pkg6_test/bin";
 
 #[derive(Parser)]
 #[command(author, version, about, long_about = None)]
@@ -44,6 +45,16 @@ enum Commands {
         package: Option<String>,
     },
     
+    /// Build binaries for end-to-end tests
+    BuildE2E,
+    
+    /// Run end-to-end tests using pre-built binaries
+    RunE2E {
+        /// Specific test to run (runs all e2e tests if not specified)
+        #[arg(short, long)]
+        test: Option<String>,
+    },
+    
     /// Format code
     Fmt,
     
@@ -61,6 +72,8 @@ fn main() -> Result<()> {
         Commands::SetupTestEnv => setup_test_env(),
         Commands::Build { release, package } => build(release, package),
         Commands::Test { release, package } => test(release, package),
+        Commands::BuildE2E => build_e2e(),
+        Commands::RunE2E { test } => run_e2e(test),
         Commands::Fmt => fmt(),
         Commands::Clippy => clippy(),
         Commands::Clean => clean(),
@@ -71,10 +84,26 @@ fn main() -> Result<()> {
 fn setup_test_env() -> Result<()> {
     println!("Setting up test environment...");
     
-    // Clean up any existing test directories
+    // Clean up any existing test directories except the bin directory
     if Path::new(TEST_BASE_DIR).exists() {
         println!("Cleaning up existing test directory...");
-        fs::remove_dir_all(TEST_BASE_DIR).context("Failed to remove existing test directory")?;
+        
+        // Remove subdirectories individually, preserving the bin directory
+        let entries = fs::read_dir(TEST_BASE_DIR).context("Failed to read test directory")?;
+        for entry in entries {
+            let entry = entry.context("Failed to read directory entry")?;
+            let path = entry.path();
+            
+            // Skip the bin directory
+            if path.is_dir() && path.file_name().unwrap_or_default() != "bin" {
+                fs::remove_dir_all(&path).context(format!("Failed to remove directory: {:?}", path))?;
+            } else if path.is_file() {
+                fs::remove_file(&path).context(format!("Failed to remove file: {:?}", path))?;
+            }
+        }
+    } else {
+        // Create the base directory if it doesn't exist
+        fs::create_dir_all(TEST_BASE_DIR).context("Failed to create test base directory")?;
     }
     
     // Create test directories
@@ -229,5 +258,84 @@ fn clean() -> Result<()> {
         .status()
         .context("Failed to clean build artifacts")?;
     
+    Ok(())
+}
+
+/// Build binaries for end-to-end tests
+fn build_e2e() -> Result<()> {
+    println!("Building binaries for end-to-end tests...");
+    
+    // Create the bin directory if it doesn't exist
+    fs::create_dir_all(E2E_TEST_BIN_DIR).context("Failed to create bin directory")?;
+    
+    // Build pkg6repo in release mode
+    println!("Building pkg6repo...");
+    Command::new("cargo")
+        .args(["build", "--release", "--package", "pkg6repo"])
+        .status()
+        .context("Failed to build pkg6repo")?;
+    
+    // Build pkg6dev in release mode
+    println!("Building pkg6dev...");
+    Command::new("cargo")
+        .args(["build", "--release", "--package", "pkg6dev"])
+        .status()
+        .context("Failed to build pkg6dev")?;
+    
+    // Copy the binaries to the bin directory
+    let target_dir = PathBuf::from("target/release");
+    
+    println!("Copying binaries to test directory...");
+    fs::copy(
+        target_dir.join("pkg6repo"),
+        PathBuf::from(E2E_TEST_BIN_DIR).join("pkg6repo"),
+    )
+    .context("Failed to copy pkg6repo binary")?;
+    
+    fs::copy(
+        target_dir.join("pkg6dev"),
+        PathBuf::from(E2E_TEST_BIN_DIR).join("pkg6dev"),
+    )
+    .context("Failed to copy pkg6dev binary")?;
+    
+    println!("End-to-end test binaries built successfully!");
+    println!("Binaries are located at: {}", E2E_TEST_BIN_DIR);
+    
+    Ok(())
+}
+
+/// Run end-to-end tests using pre-built binaries
+fn run_e2e(test: &Option<String>) -> Result<()> {
+    println!("Running end-to-end tests...");
+    
+    // Check if the binaries exist
+    let pkg6repo_bin = PathBuf::from(E2E_TEST_BIN_DIR).join("pkg6repo");
+    let pkg6dev_bin = PathBuf::from(E2E_TEST_BIN_DIR).join("pkg6dev");
+    
+    if !pkg6repo_bin.exists() || !pkg6dev_bin.exists() {
+        println!("Pre-built binaries not found. Building them first...");
+        build_e2e()?;
+    }
+    
+    // Set up the test environment
+    setup_test_env()?;
+    
+    // Run the tests
+    let mut cmd = Command::new("cargo");
+    cmd.arg("test");
+    
+    if let Some(test_name) = test {
+        cmd.args(["--package", "pkg6repo", "--test", "e2e_tests", test_name]);
+    } else {
+        cmd.args(["--package", "pkg6repo", "--test", "e2e_tests"]);
+    }
+    
+    // Set the environment variable for the test binaries
+    cmd.env("PKG6_TEST_BIN_DIR", E2E_TEST_BIN_DIR);
+    
+    cmd.status().context("Failed to run end-to-end tests")?;
+    
+    println!("End-to-end tests completed!");
+    
     Ok(())
 }