From 098c62b645303034ef3d504f846e0b9e2c9988a4 Mon Sep 17 00:00:00 2001 From: Till Wegmueller Date: Sat, 26 Jul 2025 17:49:56 +0200 Subject: [PATCH] Replace legacy `setup_test_env.sh` script with `xtask` for environment setup, refactor e2e tests to use pre-built binaries, and update `xtask` commands and documentation for improved automation and reliability. --- pkg6repo/Cargo.toml | 4 ++ pkg6repo/src/e2e_tests.rs | 96 ++++++++++++++++++++------------ setup_test_env.sh | 74 ------------------------- xtask/README.md | 78 ++++++++++++++++++++++++++ xtask/src/main.rs | 114 +++++++++++++++++++++++++++++++++++++- 5 files changed, 253 insertions(+), 113 deletions(-) delete mode 100755 setup_test_env.sh create mode 100644 xtask/README.md diff --git a/pkg6repo/Cargo.toml b/pkg6repo/Cargo.toml index dcae11b..73bb7c4 100644 --- a/pkg6repo/Cargo.toml +++ b/pkg6repo/Cargo.toml @@ -18,3 +18,7 @@ thiserror = "1.0.50" libips = { path = "../libips" } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" + +[[test]] +name = "e2e_tests" +path = "src/e2e_tests.rs" diff --git a/pkg6repo/src/e2e_tests.rs b/pkg6repo/src/e2e_tests.rs index 9c6aebc..035917d 100644 --- a/pkg6repo/src/e2e_tests.rs +++ b/pkg6repo/src/e2e_tests.rs @@ -5,6 +5,7 @@ #[cfg(test)] mod e2e_tests { + use std::env; use std::fs; use std::path::PathBuf; use std::process::Command; @@ -12,6 +13,17 @@ mod e2e_tests { // The base directory for all test repositories const TEST_REPO_BASE_DIR: &str = "/tmp/pkg6repo_e2e_test"; + + // Get the path to the pre-built binaries + fn get_bin_dir() -> PathBuf { + match env::var("PKG6_TEST_BIN_DIR") { + Ok(dir) => PathBuf::from(dir), + Err(_) => { + // Fallback to the default location if the environment variable is not set + PathBuf::from("/tmp/pkg6_test/bin") + } + } + } // Helper function to create a unique test directory fn create_test_dir(test_name: &str) -> PathBuf { @@ -35,24 +47,20 @@ mod e2e_tests { } } - // Helper function to run the setup script + // Helper function to set up the test environment fn run_setup_script() -> (PathBuf, PathBuf) { - // Get the project root directory - let output = Command::new("git") - .args(["rev-parse", "--show-toplevel"]) + // Run the xtask setup-test-env command + let output = Command::new("cargo") + .args(["run", "-p", "xtask", "--", "setup-test-env"]) .output() - .expect("Failed to execute git command"); - - let project_root = String::from_utf8(output.stdout) - .expect("Invalid UTF-8 output") - .trim() - .to_string(); - - // Run the setup script - Command::new("bash") - .arg(format!("{}/setup_test_env.sh", project_root)) - .status() - .expect("Failed to run setup script"); + .expect("Failed to run xtask setup-test-env"); + + if !output.status.success() { + panic!( + "Failed to set up test environment: {}", + String::from_utf8_lossy(&output.stderr) + ); + } // Return the paths to the prototype and manifest directories ( @@ -63,10 +71,18 @@ mod e2e_tests { // Helper function to run pkg6repo command fn run_pkg6repo(args: &[&str]) -> Result { - let output = Command::new("cargo") - .arg("run") - .arg("--bin") - .arg("pkg6repo") + let bin_dir = get_bin_dir(); + let pkg6repo_bin = bin_dir.join("pkg6repo"); + + // Check if the binary exists + if !pkg6repo_bin.exists() { + return Err(format!( + "pkg6repo binary not found at {}. Run 'cargo xtask build-e2e' first.", + pkg6repo_bin.display() + )); + } + + let output = Command::new(pkg6repo_bin) .args(args) .output() .expect("Failed to execute pkg6repo command"); @@ -80,10 +96,18 @@ mod e2e_tests { // Helper function to run pkg6dev command fn run_pkg6dev(args: &[&str]) -> Result { - let output = Command::new("cargo") - .arg("run") - .arg("--bin") - .arg("pkg6dev") + let bin_dir = get_bin_dir(); + let pkg6dev_bin = bin_dir.join("pkg6dev"); + + // Check if the binary exists + if !pkg6dev_bin.exists() { + return Err(format!( + "pkg6dev binary not found at {}. Run 'cargo xtask build-e2e' first.", + pkg6dev_bin.display() + )); + } + + let output = Command::new(pkg6dev_bin) .args(args) .output() .expect("Failed to execute pkg6dev command"); @@ -181,9 +205,9 @@ mod e2e_tests { let manifest_path = manifest_dir.join("example.p5m"); let result = run_pkg6dev(&[ "publish", - manifest_path.to_str().unwrap(), - prototype_dir.to_str().unwrap(), - repo_path.to_str().unwrap(), + "--manifest-path", manifest_path.to_str().unwrap(), + "--prototype-dir", prototype_dir.to_str().unwrap(), + "--repo-path", repo_path.to_str().unwrap(), ]); assert!( result.is_ok(), @@ -238,9 +262,9 @@ mod e2e_tests { let manifest_path = manifest_dir.join("example.p5m"); let result = run_pkg6dev(&[ "publish", - manifest_path.to_str().unwrap(), - prototype_dir.to_str().unwrap(), - repo_path.to_str().unwrap(), + "--manifest-path", manifest_path.to_str().unwrap(), + "--prototype-dir", prototype_dir.to_str().unwrap(), + "--repo-path", repo_path.to_str().unwrap(), ]); assert!( result.is_ok(), @@ -303,9 +327,9 @@ mod e2e_tests { let manifest_path1 = manifest_dir.join("example.p5m"); let result = run_pkg6dev(&[ "publish", - manifest_path1.to_str().unwrap(), - prototype_dir.to_str().unwrap(), - repo_path.to_str().unwrap(), + "--manifest-path", manifest_path1.to_str().unwrap(), + "--prototype-dir", prototype_dir.to_str().unwrap(), + "--repo-path", repo_path.to_str().unwrap(), ]); assert!( result.is_ok(), @@ -317,9 +341,9 @@ mod e2e_tests { let manifest_path2 = manifest_dir.join("example2.p5m"); let result = run_pkg6dev(&[ "publish", - manifest_path2.to_str().unwrap(), - prototype_dir.to_str().unwrap(), - repo_path.to_str().unwrap(), + "--manifest-path", manifest_path2.to_str().unwrap(), + "--prototype-dir", prototype_dir.to_str().unwrap(), + "--repo-path", repo_path.to_str().unwrap(), ]); assert!( result.is_ok(), diff --git a/setup_test_env.sh b/setup_test_env.sh deleted file mode 100755 index b7b18fb..0000000 --- a/setup_test_env.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -# Script to set up the test environment for repository tests - -set -e # Exit on error - -# Directory where test files will be created -TEST_BASE_DIR="/tmp/pkg6_test" -PROTOTYPE_DIR="$TEST_BASE_DIR/prototype" -MANIFEST_DIR="$TEST_BASE_DIR/manifests" - -# Clean up any existing test directories -if [ -d "$TEST_BASE_DIR" ]; then - echo "Cleaning up existing test directory..." - rm -rf "$TEST_BASE_DIR" -fi - -# Create test directories -echo "Creating test directories..." -mkdir -p "$PROTOTYPE_DIR" -mkdir -p "$MANIFEST_DIR" - -# Compile the applications -echo "Compiling applications..." -cd "$(dirname "$0")" -cargo build - -# Create a simple prototype directory structure with some files -echo "Creating prototype directory structure..." - -# Create some directories -mkdir -p "$PROTOTYPE_DIR/usr/bin" -mkdir -p "$PROTOTYPE_DIR/usr/share/doc/example" -mkdir -p "$PROTOTYPE_DIR/etc/config" - -# Create some files -echo "#!/bin/sh\necho 'Hello, World!'" > "$PROTOTYPE_DIR/usr/bin/hello" -chmod +x "$PROTOTYPE_DIR/usr/bin/hello" - -echo "This is an example document." > "$PROTOTYPE_DIR/usr/share/doc/example/README.txt" - -echo "# Example configuration file\nvalue=42" > "$PROTOTYPE_DIR/etc/config/example.conf" - -# Create a simple manifest -echo "Creating package manifest..." -cat > "$MANIFEST_DIR/example.p5m" << EOF -set name=pkg.fmri value=pkg://test/example@1.0.0 -set name=pkg.summary value="Example package for testing" -set name=pkg.description value="This is an example package used for testing the repository implementation." -set name=info.classification value="org.opensolaris.category.2008:System/Core" -set name=variant.arch value=i386 value=sparc -file path=usr/bin/hello mode=0755 owner=root group=bin -file path=usr/share/doc/example/README.txt mode=0644 owner=root group=bin -file path=etc/config/example.conf mode=0644 owner=root group=bin preserve=true -dir path=usr/bin mode=0755 owner=root group=bin -dir path=usr/share/doc/example mode=0755 owner=root group=bin -dir path=etc/config mode=0755 owner=root group=sys -EOF - -# Create a second manifest for testing multiple packages -cat > "$MANIFEST_DIR/example2.p5m" << EOF -set name=pkg.fmri value=pkg://test/example2@1.0.0 -set name=pkg.summary value="Second example package for testing" -set name=pkg.description value="This is a second example package used for testing the repository implementation." -set name=info.classification value="org.opensolaris.category.2008:System/Core" -set name=variant.arch value=i386 value=sparc -file path=usr/bin/hello mode=0755 owner=root group=bin -file path=usr/share/doc/example/README.txt mode=0644 owner=root group=bin -dir path=usr/bin mode=0755 owner=root group=bin -dir path=usr/share/doc/example mode=0755 owner=root group=bin -EOF - -echo "Test environment setup complete!" -echo "Prototype directory: $PROTOTYPE_DIR" -echo "Manifest directory: $MANIFEST_DIR" \ No newline at end of file diff --git a/xtask/README.md b/xtask/README.md new file mode 100644 index 0000000..cd79e62 --- /dev/null +++ b/xtask/README.md @@ -0,0 +1,78 @@ +# Xtask for IPS + +This directory contains the xtask implementation for the IPS project. Xtask is a Rust-based build system that allows us to write build scripts and automation tasks in Rust instead of shell scripts, making them more maintainable and cross-platform. + +## Available Commands + +The following commands are available through cargo-xtask: + +```bash +cargo run -p xtask -- setup-test-env # Set up the test environment for repository tests +cargo run -p xtask -- build # Build the project +cargo run -p xtask -- build -r # Build with release optimizations +cargo run -p xtask -- build -p # Build a specific crate +cargo run -p xtask -- test # Run tests +cargo run -p xtask -- test -r # Run tests with release optimizations +cargo run -p xtask -- test -p # Run tests for a specific crate +cargo run -p xtask -- build-e2e # Build binaries for end-to-end tests +cargo run -p xtask -- run-e2e # Run end-to-end tests using pre-built binaries +cargo run -p xtask -- run-e2e -t # Run a specific end-to-end test +cargo run -p xtask -- fmt # Format code using cargo fmt +cargo run -p xtask -- clippy # Run clippy for code quality checks +cargo run -p xtask -- clean # Clean build artifacts +``` + +## End-to-End Testing + +End-to-end tests are an important part of the IPS project. They test the entire system from the user's perspective, ensuring that all components work together correctly. + +### Improved End-to-End Testing Approach + +To reduce flaky tests and improve reliability, we've separated the building of binaries from the test execution. This approach has several advantages: + +1. **Reduced Flakiness**: By separating the build step from the test execution, we reduce the chance of tests failing due to build issues. +2. **Faster Test Execution**: Pre-building the binaries means that tests can start immediately without waiting for compilation. +3. **Consistent Test Environment**: Using xtask for both building binaries and setting up the test environment ensures consistency. + +### How to Run End-to-End Tests + +To run end-to-end tests, follow these steps: + +1. Build the binaries for end-to-end tests: + ```bash + cargo run -p xtask -- build-e2e + ``` + +2. Run the end-to-end tests: + ```bash + cargo run -p xtask -- run-e2e + ``` + + To run a specific test: + ```bash + cargo run -p xtask -- run-e2e -t test_e2e_create_repository + ``` + +### How It Works + +The `build-e2e` command: +- Builds the necessary binaries (pkg6repo, pkg6dev, etc.) in release mode +- Copies the binaries to a dedicated directory (`/tmp/pkg6_test/bin`) + +The `run-e2e` command: +- Checks if the pre-built binaries exist, and builds them if they don't +- Sets up the test environment using the `setup-test-env` command +- Runs the end-to-end tests, passing the location of the pre-built binaries via an environment variable + +The end-to-end tests: +- Use the pre-built binaries instead of compiling them on the fly +- Use a consistent test environment setup + +## Adding New Commands + +To add a new command to cargo-xtask: + +1. Edit the `xtask/src/main.rs` file +2. Add a new variant to the `Commands` enum +3. Implement a function for the new command +4. Add a match arm in the `main` function to call your new function \ No newline at end of file diff --git a/xtask/src/main.rs b/xtask/src/main.rs index f3f69ed..99165a1 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -2,13 +2,14 @@ use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; use std::fs::{self, File}; use std::io::Write; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::process::Command; // Constants const TEST_BASE_DIR: &str = "/tmp/pkg6_test"; const PROTOTYPE_DIR: &str = "/tmp/pkg6_test/prototype"; const MANIFEST_DIR: &str = "/tmp/pkg6_test/manifests"; +const E2E_TEST_BIN_DIR: &str = "/tmp/pkg6_test/bin"; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -44,6 +45,16 @@ enum Commands { package: Option, }, + /// Build binaries for end-to-end tests + BuildE2E, + + /// Run end-to-end tests using pre-built binaries + RunE2E { + /// Specific test to run (runs all e2e tests if not specified) + #[arg(short, long)] + test: Option, + }, + /// Format code Fmt, @@ -61,6 +72,8 @@ fn main() -> Result<()> { Commands::SetupTestEnv => setup_test_env(), Commands::Build { release, package } => build(release, package), Commands::Test { release, package } => test(release, package), + Commands::BuildE2E => build_e2e(), + Commands::RunE2E { test } => run_e2e(test), Commands::Fmt => fmt(), Commands::Clippy => clippy(), Commands::Clean => clean(), @@ -71,10 +84,26 @@ fn main() -> Result<()> { fn setup_test_env() -> Result<()> { println!("Setting up test environment..."); - // Clean up any existing test directories + // Clean up any existing test directories except the bin directory if Path::new(TEST_BASE_DIR).exists() { println!("Cleaning up existing test directory..."); - fs::remove_dir_all(TEST_BASE_DIR).context("Failed to remove existing test directory")?; + + // Remove subdirectories individually, preserving the bin directory + let entries = fs::read_dir(TEST_BASE_DIR).context("Failed to read test directory")?; + for entry in entries { + let entry = entry.context("Failed to read directory entry")?; + let path = entry.path(); + + // Skip the bin directory + if path.is_dir() && path.file_name().unwrap_or_default() != "bin" { + fs::remove_dir_all(&path).context(format!("Failed to remove directory: {:?}", path))?; + } else if path.is_file() { + fs::remove_file(&path).context(format!("Failed to remove file: {:?}", path))?; + } + } + } else { + // Create the base directory if it doesn't exist + fs::create_dir_all(TEST_BASE_DIR).context("Failed to create test base directory")?; } // Create test directories @@ -229,5 +258,84 @@ fn clean() -> Result<()> { .status() .context("Failed to clean build artifacts")?; + Ok(()) +} + +/// Build binaries for end-to-end tests +fn build_e2e() -> Result<()> { + println!("Building binaries for end-to-end tests..."); + + // Create the bin directory if it doesn't exist + fs::create_dir_all(E2E_TEST_BIN_DIR).context("Failed to create bin directory")?; + + // Build pkg6repo in release mode + println!("Building pkg6repo..."); + Command::new("cargo") + .args(["build", "--release", "--package", "pkg6repo"]) + .status() + .context("Failed to build pkg6repo")?; + + // Build pkg6dev in release mode + println!("Building pkg6dev..."); + Command::new("cargo") + .args(["build", "--release", "--package", "pkg6dev"]) + .status() + .context("Failed to build pkg6dev")?; + + // Copy the binaries to the bin directory + let target_dir = PathBuf::from("target/release"); + + println!("Copying binaries to test directory..."); + fs::copy( + target_dir.join("pkg6repo"), + PathBuf::from(E2E_TEST_BIN_DIR).join("pkg6repo"), + ) + .context("Failed to copy pkg6repo binary")?; + + fs::copy( + target_dir.join("pkg6dev"), + PathBuf::from(E2E_TEST_BIN_DIR).join("pkg6dev"), + ) + .context("Failed to copy pkg6dev binary")?; + + println!("End-to-end test binaries built successfully!"); + println!("Binaries are located at: {}", E2E_TEST_BIN_DIR); + + Ok(()) +} + +/// Run end-to-end tests using pre-built binaries +fn run_e2e(test: &Option) -> Result<()> { + println!("Running end-to-end tests..."); + + // Check if the binaries exist + let pkg6repo_bin = PathBuf::from(E2E_TEST_BIN_DIR).join("pkg6repo"); + let pkg6dev_bin = PathBuf::from(E2E_TEST_BIN_DIR).join("pkg6dev"); + + if !pkg6repo_bin.exists() || !pkg6dev_bin.exists() { + println!("Pre-built binaries not found. Building them first..."); + build_e2e()?; + } + + // Set up the test environment + setup_test_env()?; + + // Run the tests + let mut cmd = Command::new("cargo"); + cmd.arg("test"); + + if let Some(test_name) = test { + cmd.args(["--package", "pkg6repo", "--test", "e2e_tests", test_name]); + } else { + cmd.args(["--package", "pkg6repo", "--test", "e2e_tests"]); + } + + // Set the environment variable for the test binaries + cmd.env("PKG6_TEST_BIN_DIR", E2E_TEST_BIN_DIR); + + cmd.status().context("Failed to run end-to-end tests")?; + + println!("End-to-end tests completed!"); + Ok(()) } \ No newline at end of file