Lots of testing and fixing the builds to produce a Ubuntu and a omnios image

Signed-off-by: Till Wegmueller <toasterson@gmail.com>

Cargo.lock

@@ -718,6 +718,7 @@ dependencies = [
  "forge-oci",
  "libc",
  "miette 7.6.0",
+ "openssl",
  "reqwest",
  "serde_json",
  "spec-parser",
@@ -1815,6 +1816,15 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
 
+[[package]]
+name = "openssl-src"
+version = "300.5.4+3.5.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a507b3792995dae9b0df8a1c1e3771e8418b7c2d9f0baeba32e6fe8b06c7cb72"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "openssl-sys"
 version = "0.9.111"
@@ -1823,6 +1833,7 @@ checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321"
 dependencies = [
  "cc",
  "libc",
+ "openssl-src",
  "pkg-config",
  "vcpkg",
 ]
@@ -3260,6 +3271,7 @@ dependencies = [
  "libc",
  "miette 7.6.0",
  "oci-client",
+ "openssl",
  "reqwest",
  "serde",
  "serde_json",

Cargo.toml

@@ -59,6 +59,7 @@ reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-
 libc = "0.2"
 dirs = "6"
 ssh2 = "0.9"
+openssl = { version = "0.10", features = ["vendored"] }
 ssh-key = { version = "0.6", features = ["ed25519", "rand_core", "getrandom"] }
 
 # Internal crates

Cross.toml

@@ -0,0 +1,7 @@
+[build.env]
+volumes = ["VM_MANAGER_DIR=/home/toasty/ws/nebula/vm-manager"]
+
+[target.x86_64-unknown-illumos]
+pre-build = [
+    "ln -sf /usr/local/x86_64-unknown-illumos/bin/x86_64-unknown-illumos-ranlib /usr/local/bin/granlib",
+]

crates/forge-builder/Cargo.toml

@@ -14,6 +14,7 @@ tokio = { workspace = true }
 tracing = { workspace = true }
 reqwest = { workspace = true }
 ssh2 = { workspace = true }
+openssl = { workspace = true }
 ssh-key = { workspace = true }
 libc = { workspace = true }
 dirs = { workspace = true }

crates/forge-builder/src/lib.rs

@@ -133,8 +133,10 @@ fn install_build_deps(
              debootstrap qemu-utils parted dosfstools e2fsprogs grub-efi-amd64-bin mount"
         }
         DistroFamily::OmniOS => {
-            // OmniOS builder images should already have pkg tools; install qemu-img if missing
+            // OmniOS bloody: add the extra publisher for qemu-img utility
-            "sudo pkg install -q system/qemu/img || true"
+            "sudo pkg set-publisher -g https://pkg.omnios.org/bloody/extra extra.omnios 2>/dev/null; \
+             sudo pkg refresh --full 2>/dev/null; \
+             sudo pkg install -q ooce/util/qemu-img || true"
         }
     };
 
@@ -203,7 +205,7 @@ async fn run_build_in_session(
     // Build the remote command — always pass --skip-push so the VM never attempts
     // to push to the registry (it lacks GITHUB_TOKEN); the host handles pushing.
     let mut cmd = String::from(
-        "sudo /tmp/forger-build/forger build -s /tmp/forger-build/spec.kdl -o /tmp/forger-build/output/ --local --skip-push",
+        "sudo /var/tmp/forger-build/forger build -s /var/tmp/forger-build/spec.kdl -o /var/tmp/forger-build/output/ --local --skip-push",
     );
 
     if let Some(t) = target {

crates/forge-builder/src/transfer.rs

@@ -7,7 +7,7 @@ use vm_manager::ssh;
 use crate::error::BuilderError;
 use crate::lifecycle::BuilderSession;
 
-const REMOTE_BUILD_DIR: &str = "/tmp/forger-build";
+const REMOTE_BUILD_DIR: &str = "/var/tmp/forger-build";
 
 /// Upload all build inputs to the builder VM.
 pub fn upload_build_inputs(
@@ -47,6 +47,26 @@ pub fn upload_build_inputs(
         detail: format!("upload spec: {e}"),
     })?;
 
+    // Upload sibling .kdl files (base/include references resolved relative to spec dir)
+    if let Some(spec_dir) = spec_path.parent() {
+        if let Ok(entries) = std::fs::read_dir(spec_dir) {
+            for entry in entries.flatten() {
+                let path = entry.path();
+                if path.extension().is_some_and(|e| e == "kdl") && path != spec_path {
+                    let filename = path.file_name().unwrap();
+                    let remote_path =
+                        PathBuf::from(format!("{REMOTE_BUILD_DIR}/{}", filename.to_string_lossy()));
+                    info!(file = %filename.to_string_lossy(), "Uploading include file");
+                    ssh::upload(sess, &path, &remote_path).map_err(|e| {
+                        BuilderError::TransferFailed {
+                            detail: format!("upload include {}: {e}", filename.to_string_lossy()),
+                        }
+                    })?;
+                }
+            }
+        }
+    }
+
     // Upload files/ directory if it exists (tar locally → upload → extract remotely)
     if files_dir.exists() && files_dir.is_dir() {
         upload_directory(sess, files_dir, &format!("{REMOTE_BUILD_DIR}/files"))?;
@@ -109,10 +129,11 @@ pub fn download_artifacts(
     let sess = &session.ssh_session;
     let remote_output = format!("{REMOTE_BUILD_DIR}/output");
 
-    // List files in remote output directory
+    // List files in remote output directory (use ls -1 for portability; GNU
+    // find -printf is not available on illumos)
     let (stdout, _, exit_code) = ssh::exec(
         sess,
-        &format!("find {remote_output} -maxdepth 1 -type f -printf '%f\\n'"),
+        &format!("ls -1 {remote_output}/ 2>/dev/null"),
     )
     .map_err(|e| BuilderError::DownloadFailed {
         detail: format!("list remote files: {e}"),

crates/forge-engine/src/phase2/qcow2_zfs.rs

@@ -12,7 +12,10 @@ pub struct PreparedZfs {
     pub raw_path: PathBuf,
     pub qcow2_path: PathBuf,
     pub device: String,
+    /// Build-time pool name (unique to avoid collision with host's rpool).
     pub pool_name: String,
+    /// Final pool name for the output image (typically "rpool").
+    pub final_pool_name: String,
     pub be_dataset: String,
     pub bootloader_type: String,
     pub mount_dir: tempfile::TempDir,
@@ -58,7 +61,16 @@ pub async fn prepare_zfs(
         })
         .unwrap_or_default();
 
-    let pool_name = "rpool".to_string();
+    // Use a unique build-time pool name to avoid collisions when building
+    // inside a VM that already has its own "rpool" (e.g. OmniOS builder VMs).
+    // The pool is renamed to the final name after export.
+    let final_pool_name = "rpool".to_string();
+    let nanos = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap_or_default()
+        .subsec_nanos();
+    let build_id = format!("{:08x}", nanos ^ std::process::id());
+    let pool_name = format!("forgebuild_{build_id}");
     let be_dataset = format!("{pool_name}/ROOT/be-1");
 
     info!(disk_size, "Step 1: Creating raw disk image");
@@ -98,13 +110,15 @@ pub async fn prepare_zfs(
         qcow2_path,
         device,
         pool_name,
+        final_pool_name,
         be_dataset,
         bootloader_type,
         mount_dir,
     })
 }
 
-/// Phase 2 finalize: install bootloader, set bootfs, unmount + export pool.
+/// Phase 2 finalize: install bootloader, set bootfs, unmount + export pool,
+/// then rename the build-time pool to the final name (e.g. "rpool").
 pub async fn finalize_zfs(
     prepared: &PreparedZfs,
     runner: &dyn ToolRunner,
@@ -127,6 +141,41 @@ pub async fn finalize_zfs(
     crate::tools::zfs::unmount(runner, &prepared.be_dataset).await?;
     crate::tools::zpool::export(runner, &prepared.pool_name).await?;
 
+    // Rename the pool from the build-time name to the final name.
+    // The pool is exported and the loopback device is still attached,
+    // so we can reimport with the new name.
+    //
+    // This will fail inside builder VMs that have their own "rpool" active
+    // (e.g. OmniOS builders) — in that case the image keeps the build-time
+    // pool name and can be renamed at deployment with:
+    //   zpool import <build_name> rpool
+    if prepared.pool_name != prepared.final_pool_name {
+        info!(
+            build_name = %prepared.pool_name,
+            final_name = %prepared.final_pool_name,
+            "Finalize step 4: Renaming pool to final name"
+        );
+        match crate::tools::zpool::rename_exported(
+            runner,
+            &prepared.device,
+            &prepared.pool_name,
+            &prepared.final_pool_name,
+        )
+        .await
+        {
+            Ok(()) => info!("Pool renamed to '{}'", prepared.final_pool_name),
+            Err(e) => tracing::warn!(
+                error = %e,
+                build_name = %prepared.pool_name,
+                "Pool rename failed (host likely has active '{pool}') — \
+                 image pool is named '{build}'; rename at deployment with: \
+                 zpool import {build} {pool}",
+                pool = prepared.final_pool_name,
+                build = prepared.pool_name,
+            ),
+        }
+    }
+
     Ok(())
 }
 

crates/forge-engine/src/tools/pkg.rs

@@ -5,7 +5,7 @@ use tracing::info;
 /// Create a new IPS image at the given root path.
 pub async fn image_create(runner: &dyn ToolRunner, root: &str) -> Result<(), ForgeError> {
     info!(root, "Creating IPS image");
-    runner.run("pkg", &["image-create", "-F", "-p", root]).await?;
+    runner.run("pkg", &["image-create", "-F", root]).await?;
     Ok(())
 }
 
@@ -41,6 +41,8 @@ pub async fn install(
 }
 
 /// Change an IPS variant in the image at the given root.
+///
+/// Exit code 4 from `pkg` means "nothing to do" (already set) — treated as success.
 pub async fn change_variant(
     runner: &dyn ToolRunner,
     root: &str,
@@ -49,10 +51,17 @@ pub async fn change_variant(
 ) -> Result<(), ForgeError> {
     info!(root, name, value, "Changing variant");
     let variant_arg = format!("{name}={value}");
-    runner
+    match runner
         .run("pkg", &["-R", root, "change-variant", &variant_arg])
-        .await?;
+        .await
+    {
+        Ok(_) => Ok(()),
+        Err(ForgeError::ToolNonZero { exit_code: 4, .. }) => {
+            info!(name, value, "Variant already set — nothing to do");
             Ok(())
+        }
+        Err(e) => Err(e),
+    }
 }
 
 /// Approve a CA certificate for a publisher in the IPS image.

crates/forge-engine/src/tools/zpool.rs

@@ -12,6 +12,9 @@ pub async fn create(
     info!(pool_name, device, "Creating ZFS pool");
     let mut args = vec!["create"];
 
+    // Suppress default mountpoint — child datasets set explicit mountpoints
+    args.extend_from_slice(&["-m", "none"]);
+
     // Add -o property=value for each pool property
     let prop_strings: Vec<String> = properties
         .iter()
@@ -36,6 +39,34 @@ pub async fn export(runner: &dyn ToolRunner, pool_name: &str) -> Result<(), Forg
     Ok(())
 }
 
+/// Rename an exported pool by importing it with a new name and re-exporting.
+///
+/// The pool must already be exported. The `device` is the loopback device
+/// (e.g. `/dev/lofi/1`) where the pool resides — used with `-d` to avoid
+/// scanning all devices.
+pub async fn rename_exported(
+    runner: &dyn ToolRunner,
+    device: &str,
+    old_name: &str,
+    new_name: &str,
+) -> Result<(), ForgeError> {
+    info!(old_name, new_name, device, "Renaming ZFS pool via import/export");
+
+    // Import from the specific device, rename, don't mount anything
+    runner
+        .run(
+            "zpool",
+            &["import", "-f", "-N", "-d", device, old_name, new_name],
+        )
+        .await?;
+
+    // Immediately export the renamed pool
+    runner.run("zpool", &["export", new_name]).await?;
+
+    info!(new_name, "Pool renamed successfully");
+    Ok(())
+}
+
 /// Destroy a ZFS pool (force).
 pub async fn destroy(runner: &dyn ToolRunner, pool_name: &str) -> Result<(), ForgeError> {
     info!(pool_name, "Destroying ZFS pool");

images/files/omniosce-ca.cert.pem

@@ -0,0 +1,35 @@
+-----BEGIN CERTIFICATE-----
+MIIGGDCCBACgAwIBAgIJAL31YgRC8LEyMA0GCSqGSIb3DQEBCwUAMH4xCzAJBgNV
+BAYTAkNIMQ4wDAYDVQQHDAVPbHRlbjEhMB8GA1UECgwYT21uaU9TIENvbW11bml0
+eSBFZGl0aW9uMRwwGgYDVQQDDBNPbW5pT1NjZSBLZXkgTWFzdGVyMR4wHAYJKoZI
+hvcNAQkBFg9jYUBvbW5pb3NjZS5vcmcwHhcNMTcwNzEwMDkzOTEzWhcNMzcwNzA1
+MDkzOTEzWjB+MQswCQYDVQQGEwJDSDEOMAwGA1UEBwwFT2x0ZW4xITAfBgNVBAoM
+GE9tbmlPUyBDb21tdW5pdHkgRWRpdGlvbjEcMBoGA1UEAwwTT21uaU9TY2UgS2V5
+IE1hc3RlcjEeMBwGCSqGSIb3DQEJARYPY2FAb21uaW9zY2Uub3JnMIICIjANBgkq
+hkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1GLAkBLx7aliR++b2Pv4KOyq4+rb8M8y
+GcEpr6cSOpg2sVyTQMA5J/g/6Afveay+SwRd13vnWzHKvqBbHljxMuIZSxtAehet
+aSmUmMKi+LvnaG6XSkbYsnoNlo4TZ7hCV1tlIowG1UBmdp5xYo1D4bIE4abDD++2
+S1F1j1+edT97GNaXN61zb7jhmvG7UUD51QC5DNcLss2JHqB4lmWzn0zUUotpeSjJ
+3NAnNWKqRFBJ91Wv9/NTOuVzOnV2g1n9boO7cCikgmLzWsq2HF8vTJOuBpce4G0y
+cpuBkMPDbVD2p1b4ikblKPUdOaoleglwaePVloxjtPy5VlTejsvEnEyOGfTHYRsY
+BcyAKJmyC0iAwAKRCk1JkgJCEm41Gr15SpV9xojSJyf8bUPC1PhKpCy6sCMkn8yl
+oZugzE8pjksPJ4WnJ1kVCIv06KzVq8eGkuJVV6QK/gEj2CW8J8VCN/npm47+NsP1
+YU3P/yx6aikOj16vN3f0Q9flnYaiH+o4f15PvIdbhL2AHwj2hZWgY+YkUg+/+aH0
+euMYCmr9cjRtrr0F3Kp+Mt0wwp6EHfNdZqki3Ad62s9vwgFMO43VTF8pRRVCElZV
+OgpqddGEY1TRO+Fuwh5oZJwh4UgtUHCvUWnU4dGmsNvj9RGgWy06bkV1ESYdsXY0
+/JTTL+xXcZUCAwEAAaOBmDCBlTAdBgNVHQ4EFgQUfaD8ilVITKVYtHM3Daz73cHf
+gLMwHwYDVR0jBBgwFoAUfaD8ilVITKVYtHM3Daz73cHfgLMwDwYDVR0TAQH/BAUw
+AwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRwczovL2NybC5vbW5pb3NjZS5vcmcv
+cm9vdC5jcmwwDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEBCwUAA4ICAQAztPu0
+gZ9Ro7qfarvhF5A2U8/fZyeu4KfeCWfyNttNsaZgY6E3kMLTRAvpnKDFbuJg5TIb
+yjISZI2nD9QFOr1FSP4X8xRunsW3BdXrjo7Lr/Z3UDiJ2LpsEB41i9n9EB+TcCEo
+Bln8gW/RtDUAcapbt11fP4y8795lQ18fyOqzkTLsoEWNnTdo0QMGbWhK6iJes57f
+zidMz421zIdorS2rDfgvvgkLxxXFSUhxnO206Aj8V8Gjv5PkJR4Vptk5BITRjr2B
+3a4Xhl1iBVsG8BxSgQF97HdOHkVm5yU2gQlWyzL07XoCSVhnkNebPORUrzqEC33g
+mfp9rm9XmOFLd+lCli5TvDqLO1hPAE3DHkcZ2nN02I8TLTBxM15lwA+NOLOzmdSI
+piaQd/jRF/b8l083MPKp37Zc++LVN/F3CeJX8eJuwMPyGpni4qwi2W1su4hDR/1q
+S948zyjgb8uJtCIhXiG6UvmQ7kw+8Z5vOI72f7SeYesIb/H8xGgvG0oiq4sAZ/ea
+jdR3g+dBOj8iMkvsKU9I0Vhu56nppgA0KZMKmZZyFugSo5oWgtQfp8iljdHt0YmL
+6NKGPrD1eL3Z5bxeh0F3fqmB6feDWpPDDlDXiyCzuXWVnll8hj4E3N8pCEusVtAd
+6oFE4DOg6TH3atBGbqE1Yh3kMLKJof2Ftjwh9w==
+-----END CERTIFICATE-----

images/omnios-rust-ci.kdl

@@ -13,7 +13,7 @@ packages {
     package "/driver/network/vioif"
     package "/driver/storage/vioblk"
     package "/developer/build-essential"
-    package "/developer/lang/rust"
+    package "ooce/developer/rust"
     package "/developer/versioning/git"
 }
 
@@ -23,13 +23,13 @@ overlays {
 }
 
 builder {
-    image "https://downloads.omnios.org/media/bloody/omnios-bloody-cloud.raw.zst"
+    image "https://downloads.omnios.org/media/bloody/omnios-bloody-20251111.cloud.raw.zst"
     vcpus 4
     memory 4096
 }
 
 target "qcow2" kind="qcow2" {
-    disk-size "4000M"
+    disk-size "8G"
     bootloader "uefi"
     filesystem "zfs"
     push-to "ghcr.io/cloudnebulaproject/omnios-rust:latest"

images/ubuntu-rust-ci.kdl

@@ -17,6 +17,7 @@ packages {
     package "libssl-dev"
     package "openssh-server"
     package "cloud-init"
+    package "cloud-guest-utils"
     package "grub-efi-amd64"
     package "linux-image-generic"
 }