From c9fc05a00ebf7af101bd06dc64c3b52238278319525f46370c390a0e46db121a Mon Sep 17 00:00:00 2001 From: Till Wegmueller Date: Tue, 7 Apr 2026 15:56:10 +0200 Subject: [PATCH] Remove libvirt dependencies and clean up orchestrator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove `virt` crate dependency and libvirt feature flag - Remove `ssh2` crate dependency (vm-manager handles SSH) - Remove `zstd` crate dependency (vm-manager handles decompression) - Remove LibvirtHypervisor, ZonesHypervisor, RouterHypervisor from hypervisor.rs - Remove libvirt error types from error.rs - Remove libvirt_uri/libvirt_network CLI options, add network_bridge - Replace RouterHypervisor::build() with VmManagerAdapter::build() - Update deb package depends: libvirt → qemu-system-x86 - Keep Noop backend for development/testing - Dead old SSH/console functions left for future cleanup --- crates/orchestrator/Cargo.toml | 17 +- crates/orchestrator/src/error.rs | 71 --- crates/orchestrator/src/hypervisor.rs | 615 +------------------------- crates/orchestrator/src/main.rs | 21 +- crates/orchestrator/src/scheduler.rs | 2 - 5 files changed, 14 insertions(+), 712 deletions(-) diff --git a/crates/orchestrator/Cargo.toml b/crates/orchestrator/Cargo.toml index a987b51..1319c9f 100644 --- a/crates/orchestrator/Cargo.toml +++ b/crates/orchestrator/Cargo.toml @@ -5,8 +5,6 @@ edition = "2024" build = "build.rs" [features] -# Enable libvirt backend on Linux hosts (uses virt crate on Linux) -libvirt = [] [dependencies] common = { path = "../common" } @@ -24,12 +22,9 @@ config = { version = "0.15", default-features = false, features = ["yaml"] } reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "http2", "gzip", "brotli", "zstd"] } # HTTP server for logs (runner serving removed) axum = { version = "0.8", features = ["macros"] } -# SSH client for upload/exec/logs -ssh2 = "0.9" +# SSH key generation (vm-manager handles SSH connections) ssh-key = { version = "0.6", features = ["ed25519"] } rand_core = "0.6" -# Compression/decompression -zstd = "0.13" # DB (optional basic persistence) sea-orm = { version = "1.1.17", default-features = false, features = ["sqlx-postgres", "sqlx-sqlite", "runtime-tokio-rustls", "macros", "with-uuid", "with-chrono" ] } migration = { path = "../migration" } @@ -40,9 +35,6 @@ dashmap = "6" async-trait = "0.1" uuid = { version = "1", features = ["v4", "serde"] } -[target.'cfg(target_os = "linux")'.dependencies] -virt = { version = "0.4.3" } - [package.metadata.deb] name = "solstice-orchestrator" maintainer = "Solstice CI " @@ -55,11 +47,10 @@ assets = [ ["../../examples/etc/solstice/orchestrator.env.sample", "/etc/solstice/orchestrator.env", "640"], ] depends = [ - "libvirt-daemon-system", - "libvirt-clients", + "qemu-system-x86", + "qemu-utils", "ca-certificates", - "openssh-client", ] -recommends = ["qemu-kvm", "virtinst"] +recommends = ["qemu-kvm"] conf-files = ["/etc/solstice/orchestrator.yaml", "/etc/solstice/orchestrator.env"] maintainer-scripts = "packaging/debian/" diff --git a/crates/orchestrator/src/error.rs b/crates/orchestrator/src/error.rs index b733ad5..7fc1b31 100644 --- a/crates/orchestrator/src/error.rs +++ b/crates/orchestrator/src/error.rs @@ -2,70 +2,6 @@ use thiserror::Error; #[derive(Debug, Error)] pub enum OrchestratorError { - #[error("libvirt connect failed: {0}")] - LibvirtConnect(#[source] anyhow::Error), - - #[error("define domain failed: {0}")] - LibvirtDefine(#[source] anyhow::Error), - - #[error("lookup domain failed: {0}")] - LibvirtLookup(#[source] anyhow::Error), - - #[error("domain operation failed: {0}")] - LibvirtDomain(#[source] anyhow::Error), - - #[error("qemu-img not found or failed: {0}")] - QemuImg(#[source] anyhow::Error), - - #[error("qemu-img convert failed: {0}")] - QemuImgConvert(#[source] anyhow::Error), - - #[error("parse qemu-img info json failed: {0}")] - QemuImgParse(#[source] anyhow::Error), - - #[error("mkisofs/genisoimage not found: {0}")] - MkIsoFs(#[source] anyhow::Error), - - #[error("SSH session new failed")] - SshSessionNew, - - #[error("tcp connect failed: {0}")] - TcpConnect(#[source] std::io::Error), - - #[error("ssh handshake: {0}")] - SshHandshake(#[source] ssh2::Error), - - #[error("ssh auth failed for {user}: {source}")] - SshAuth { - user: String, - #[source] - source: ssh2::Error, - }, - - #[error("ssh not authenticated")] - SshNotAuthenticated, - - #[error("sftp init failed: {0}")] - SftpInit(#[source] ssh2::Error), - - #[error("open local runner: {0}")] - OpenLocalRunner(#[source] std::io::Error), - - #[error("read runner: {0}")] - ReadRunner(#[source] std::io::Error), - - #[error("sftp create: {0}")] - SftpCreate(#[source] ssh2::Error), - - #[error("sftp write: {0}")] - SftpWrite(#[source] std::io::Error), - - #[error("channel session: {0}")] - ChannelSession(#[source] ssh2::Error), - - #[error("exec: {0}")] - Exec(#[source] ssh2::Error), - #[error("ssh keygen failed: {0}")] SshKeygen(#[source] anyhow::Error), @@ -81,10 +17,3 @@ pub enum OrchestratorError { #[error("failed to apply migrations: {0}")] DbMigrate(#[source] anyhow::Error), } - -// Helper conversions for common external error types into anyhow::Error where needed. -impl From for OrchestratorError { - fn from(e: virt::error::Error) -> Self { - OrchestratorError::LibvirtDomain(anyhow::Error::new(e)) - } -} diff --git a/crates/orchestrator/src/hypervisor.rs b/crates/orchestrator/src/hypervisor.rs index e41d37f..7455cee 100644 --- a/crates/orchestrator/src/hypervisor.rs +++ b/crates/orchestrator/src/hypervisor.rs @@ -1,19 +1,14 @@ use async_trait::async_trait; use miette::{IntoDiagnostic as _, Result}; -use std::os::unix::prelude::PermissionsExt; use std::{path::PathBuf, time::Duration}; use tracing::info; -// Backend tag is used internally to remember which backend handled this VM. +/// Backend tag is used internally to remember which backend handled this VM. #[derive(Debug, Clone, Copy)] pub enum BackendTag { Noop, Qemu, Propolis, - #[cfg(all(target_os = "linux", feature = "libvirt"))] - Libvirt, - #[cfg(target_os = "illumos")] - Zones, } #[derive(Debug, Clone)] @@ -77,176 +72,11 @@ pub trait Hypervisor: Send + Sync { async fn state(&self, _vm: &VmHandle) -> Result { Ok(VmState::Prepared) } - async fn guest_ip(&self, vm: &VmHandle) -> Result { + async fn guest_ip(&self, _vm: &VmHandle) -> Result { Ok("127.0.0.1".to_string()) } } -/// A router that delegates to the correct backend implementation per job. -pub struct RouterHypervisor { - pub noop: NoopHypervisor, - #[cfg(all(target_os = "linux", feature = "libvirt"))] - pub libvirt: Option, - #[cfg(target_os = "illumos")] - pub zones: Option, -} - -impl RouterHypervisor { - #[allow(unused_variables)] - pub fn build(libvirt_uri: String, libvirt_network: String) -> Self { - #[cfg(all(target_os = "linux", feature = "libvirt"))] - { - return RouterHypervisor { - noop: NoopHypervisor::default(), - libvirt: Some(LibvirtHypervisor { - uri: libvirt_uri, - network: libvirt_network, - }), - }; - } - #[cfg(target_os = "illumos")] - { - return RouterHypervisor { - noop: NoopHypervisor::default(), - zones: Some(ZonesHypervisor), - }; - } - #[cfg(all( - not(target_os = "illumos"), - not(all(target_os = "linux", feature = "libvirt")) - ))] - { - return RouterHypervisor { - noop: NoopHypervisor::default(), - }; - } - } -} - -#[async_trait] -impl Hypervisor for RouterHypervisor { - async fn prepare(&self, spec: &VmSpec, ctx: &JobContext) -> Result { - #[cfg(all(target_os = "linux", feature = "libvirt"))] - { - if let Some(ref hv) = self.libvirt { - return hv.prepare(spec, ctx).await; - } - } - #[cfg(target_os = "illumos")] - { - if let Some(ref hv) = self.zones { - return hv.prepare(spec, ctx).await; - } - } - self.noop.prepare(spec, ctx).await - } - async fn start(&self, vm: &VmHandle) -> Result<()> { - match vm.backend { - #[cfg(all(target_os = "linux", feature = "libvirt"))] - BackendTag::Libvirt => { - if let Some(ref hv) = self.libvirt { - hv.start(vm).await - } else { - self.noop.start(vm).await - } - } - #[cfg(target_os = "illumos")] - BackendTag::Zones => { - if let Some(ref hv) = self.zones { - hv.start(vm).await - } else { - self.noop.start(vm).await - } - } - _ => self.noop.start(vm).await, - } - } - async fn stop(&self, vm: &VmHandle, t: Duration) -> Result<()> { - match vm.backend { - #[cfg(all(target_os = "linux", feature = "libvirt"))] - BackendTag::Libvirt => { - if let Some(ref hv) = self.libvirt { - hv.stop(vm, t).await - } else { - self.noop.stop(vm, t).await - } - } - #[cfg(target_os = "illumos")] - BackendTag::Zones => { - if let Some(ref hv) = self.zones { - hv.stop(vm, t).await - } else { - self.noop.stop(vm, t).await - } - } - _ => self.noop.stop(vm, t).await, - } - } - async fn suspend(&self, vm: &VmHandle) -> Result<()> { - match vm.backend { - #[cfg(all(target_os = "linux", feature = "libvirt"))] - BackendTag::Libvirt => { - if let Some(ref hv) = self.libvirt { - hv.suspend(vm).await - } else { - self.noop.suspend(vm).await - } - } - #[cfg(target_os = "illumos")] - BackendTag::Zones => { - if let Some(ref hv) = self.zones { - hv.suspend(vm).await - } else { - self.noop.suspend(vm).await - } - } - _ => self.noop.suspend(vm).await, - } - } - async fn destroy(&self, vm: VmHandle) -> Result<()> { - match vm.backend { - #[cfg(all(target_os = "linux", feature = "libvirt"))] - BackendTag::Libvirt => { - if let Some(ref hv) = self.libvirt { - hv.destroy(vm).await - } else { - self.noop.destroy(vm).await - } - } - #[cfg(target_os = "illumos")] - BackendTag::Zones => { - if let Some(ref hv) = self.zones { - hv.destroy(vm).await - } else { - self.noop.destroy(vm).await - } - } - _ => self.noop.destroy(vm).await, - } - } - async fn state(&self, vm: &VmHandle) -> Result { - match vm.backend { - #[cfg(all(target_os = "linux", feature = "libvirt"))] - BackendTag::Libvirt => { - if let Some(ref hv) = self.libvirt { - hv.state(vm).await - } else { - Ok(VmState::Prepared) - } - } - #[cfg(target_os = "illumos")] - BackendTag::Zones => { - if let Some(ref hv) = self.zones { - hv.state(vm).await - } else { - Ok(VmState::Prepared) - } - } - _ => Ok(VmState::Prepared), - } - } -} - /// No-op hypervisor for development on hosts without privileges. #[derive(Debug, Clone, Default)] pub struct NoopHypervisor; @@ -288,444 +118,3 @@ impl Hypervisor for NoopHypervisor { Ok(()) } } - -#[cfg(all(target_os = "linux", feature = "libvirt"))] -pub struct LibvirtHypervisor { - pub uri: String, - pub network: String, -} - -#[cfg(all(target_os = "linux", feature = "libvirt"))] -impl LibvirtHypervisor { - fn mk_work_dir(&self, id: &str) -> std::path::PathBuf { - // Prefer /var/lib/solstice-ci if writable, else tmp - let base = std::path::Path::new("/var/lib/solstice-ci"); - let dir = if base.exists() && base.is_dir() && std::fs::metadata(base).is_ok() { - base.join(id) - } else { - std::env::temp_dir().join("solstice-libvirt").join(id) - }; - let _ = std::fs::create_dir_all(&dir); - // Make directory broadly accessible so host qemu (libvirt) can create/read files - let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o777)); - dir - } -} - -#[cfg(all(target_os = "linux", feature = "libvirt"))] -#[async_trait] -impl Hypervisor for LibvirtHypervisor { - async fn prepare(&self, spec: &VmSpec, ctx: &JobContext) -> Result { - use std::process::Command; - - let id = format!("job-{}", ctx.request_id); - let work_dir = self.mk_work_dir(&id); - - // Ensure network is active via virt crate; best-effort - let uri = self.uri.clone(); - let net_name = self.network.clone(); - tokio::task::spawn_blocking(move || -> miette::Result<()> { - use virt::{connect::Connect, network::Network}; - let conn = Connect::open(Some(&uri)) - .map_err(|e| miette::miette!("libvirt connect failed: {e}"))?; - if let Ok(net) = Network::lookup_by_name(&conn, &net_name) { - // If not active, try to create (activate). Then set autostart. - let active = net.is_active().unwrap_or(false); - if !active { - let _ = net.create(); - } - let _ = net.set_autostart(true); - } - Ok(()) - }) - .await - .into_diagnostic()??; - - // Create qcow2 overlay - let overlay = work_dir.join("overlay.qcow2"); - let size_arg = format!("{}G", spec.disk_gb); - let status = tokio::task::spawn_blocking({ - let base = spec.image_path.clone(); - let overlay = overlay.clone(); - move || -> miette::Result<()> { - // Detect base image format to set -F accordingly (raw or qcow2) - let base_fmt_out = std::process::Command::new("qemu-img") - .args(["info", "--output=json"]) - .arg(&base) - .output() - .map_err(|e| miette::miette!("qemu-img not found or failed: {e}"))?; - if !base_fmt_out.status.success() { - return Err(miette::miette!( - "qemu-img info failed: {}", - String::from_utf8_lossy(&base_fmt_out.stderr) - )); - } - let base_fmt: String = { - let v: serde_json::Value = serde_json::from_slice(&base_fmt_out.stdout) - .map_err(|e| miette::miette!("parse qemu-img info json failed: {e}"))?; - v.get("format") - .and_then(|f| f.as_str()) - .unwrap_or("raw") - .to_string() - }; - - let out = Command::new("qemu-img") - .args(["create", "-f", "qcow2", "-F"]) - .arg(&base_fmt) - .args(["-b"]) - .arg(&base) - .arg(&overlay) - .arg(&size_arg) - .output() - .map_err(|e| miette::miette!("qemu-img not found or failed: {e}"))?; - if !out.status.success() { - return Err(miette::miette!( - "qemu-img create failed: {}", - String::from_utf8_lossy(&out.stderr) - )); - } - Ok(()) - } - }) - .await - .into_diagnostic()??; - let _ = status; // appease compiler if unused - // Relax permissions on overlay so host qemu can access it - let _ = std::fs::set_permissions(&overlay, std::fs::Permissions::from_mode(0o666)); - - // Build NoCloud seed ISO if user_data provided, or synthesize from per-job SSH key - let mut seed_iso: Option = None; - // Prefer spec-provided user-data; otherwise, if we have a per-job SSH public key, generate minimal cloud-config - let user_data_opt: Option> = if let Some(ref ud) = spec.user_data { - Some(ud.clone()) - } else if let Some(ref pubkey) = ctx.ssh_public_key { - let s = format!( - r#"#cloud-config -users: - - default -ssh_authorized_keys: - - {ssh_pubkey} -"#, - ssh_pubkey = pubkey.trim() - ); - Some(s.into_bytes()) - } else { - None - }; - - if let Some(user_data) = user_data_opt { - let seed_dir = work_dir.join("seed"); - tokio::fs::create_dir_all(&seed_dir) - .await - .into_diagnostic()?; - let ud_path = seed_dir.join("user-data"); - let md_path = seed_dir.join("meta-data"); - tokio::fs::write(&ud_path, &user_data) - .await - .into_diagnostic()?; - let meta = format!("instance-id: {}\nlocal-hostname: {}\n", id, id); - tokio::fs::write(&md_path, meta.as_bytes()) - .await - .into_diagnostic()?; - - // mkisofs or genisoimage - let iso_path = work_dir.join("seed.iso"); - tokio::task::spawn_blocking({ - let iso_path = iso_path.clone(); - let seed_dir = seed_dir.clone(); - move || -> miette::Result<()> { - let try_mk = |bin: &str| -> std::io::Result { - Command::new(bin) - .args(["-V", "cidata", "-J", "-R", "-o"]) - .arg(&iso_path) - .arg(&seed_dir) - .output() - }; - let out = try_mk("mkisofs") - .or_else(|_| try_mk("genisoimage")) - .map_err(|e| miette::miette!("mkisofs/genisoimage not found: {e}"))?; - if !out.status.success() { - return Err(miette::miette!( - "mkisofs failed: {}", - String::from_utf8_lossy(&out.stderr) - )); - } - Ok(()) - } - }) - .await - .into_diagnostic()??; - seed_iso = Some(iso_path); - } - - // Relax permissions on seed ISO if created (readable by host qemu) - if let Some(ref p) = seed_iso { - let _ = std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o644)); - } - - // Serial console log file path (pre-create with permissive perms for libvirt) - let console_log = work_dir.join("console.log"); - let _ = std::fs::OpenOptions::new() - .create(true) - .append(true) - .open(&console_log); - let _ = std::fs::set_permissions(&console_log, std::fs::Permissions::from_mode(0o666)); - let console_log_str = console_log.display().to_string(); - info!(domain = %id, console = %console_log_str, "serial console will be logged to file"); - - // Domain XML - let xml = { - let mem = spec.ram_mb; - let vcpus = spec.cpu; - let overlay_str = overlay.display().to_string(); - let seed_str = seed_iso.as_ref().map(|p| p.display().to_string()); - let net = self.network.clone(); - let cdrom = seed_str.map(|p| format!("\n \n \n \n \n", p)).unwrap_or_default(); - format!( - "\n{}\n{}\n{}\n\n hvm\n \n\n\n\n \n \n \n \n \n {}\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\ndestroy\ndestroy\n", - id, mem, vcpus, overlay_str, cdrom, net, console_log_str - ) - }; - - // Define via virt crate - let uri2 = self.uri.clone(); - let xml_clone = xml.clone(); - tokio::task::spawn_blocking(move || -> miette::Result<()> { - use virt::{connect::Connect, domain::Domain}; - let conn = Connect::open(Some(&uri2)) - .map_err(|e| miette::miette!("libvirt connect failed: {e}"))?; - let _dom = Domain::define_xml(&conn, &xml_clone) - .map_err(|e| miette::miette!("define domain failed: {e}"))?; - Ok(()) - }) - .await - .into_diagnostic()??; - - info!(domain = %id, image = ?spec.image_path, cpu = spec.cpu, ram_mb = spec.ram_mb, "libvirt prepared"); - Ok(VmHandle { - id, - backend: BackendTag::Libvirt, - work_dir, - overlay_path: Some(overlay), - seed_iso_path: seed_iso, - console_socket: None, - ssh_host_port: None, - mac_addr: None, - }) - } - - async fn start(&self, vm: &VmHandle) -> Result<()> { - let id = vm.id.clone(); - let uri = self.uri.clone(); - tokio::task::spawn_blocking(move || -> miette::Result<()> { - use virt::{connect::Connect, domain::Domain}; - let conn = Connect::open(Some(&uri)) - .map_err(|e| miette::miette!("libvirt connect failed: {e}"))?; - // Lookup domain by name and start - let dom = Domain::lookup_by_name(&conn, &id) - .map_err(|e| miette::miette!("lookup domain failed: {e}"))?; - dom.create() - .map_err(|e| miette::miette!("domain start failed: {e}"))?; - Ok(()) - }) - .await - .into_diagnostic()??; - info!(domain = %vm.id, "libvirt started"); - Ok(()) - } - - async fn stop(&self, vm: &VmHandle, t: Duration) -> Result<()> { - let id = vm.id.clone(); - let uri = self.uri.clone(); - tokio::task::spawn_blocking(move || -> miette::Result<()> { - use virt::{connect::Connect, domain::Domain}; - let conn = Connect::open(Some(&uri)) - .map_err(|e| miette::miette!("libvirt connect failed: {e}"))?; - let dom = Domain::lookup_by_name(&conn, &id) - .map_err(|e| miette::miette!("lookup domain failed: {e}"))?; - let _ = dom.shutdown(); - let start = std::time::Instant::now(); - while start.elapsed() < t { - match dom.is_active() { - Ok(false) => break, - _ => {} - } - std::thread::sleep(std::time::Duration::from_millis(500)); - } - // Force destroy if still active - let _ = dom.destroy(); - Ok(()) - }) - .await - .into_diagnostic()??; - info!(domain = %vm.id, "libvirt stopped"); - Ok(()) - } - async fn suspend(&self, vm: &VmHandle) -> Result<()> { - let id = vm.id.clone(); - let uri = self.uri.clone(); - tokio::task::spawn_blocking(move || -> miette::Result<()> { - use virt::{connect::Connect, domain::Domain}; - let conn = Connect::open(Some(&uri)) - .map_err(|e| miette::miette!("libvirt connect failed: {e}"))?; - let dom = Domain::lookup_by_name(&conn, &id) - .map_err(|e| miette::miette!("lookup domain failed: {e}"))?; - dom.suspend() - .map_err(|e| miette::miette!("domain suspend failed: {e}"))?; - Ok(()) - }) - .await - .into_diagnostic()??; - info!(domain = %vm.id, "libvirt suspended"); - Ok(()) - } - - async fn destroy(&self, vm: VmHandle) -> Result<()> { - let id = vm.id.clone(); - let uri = self.uri.clone(); - let id_for_task = id.clone(); - tokio::task::spawn_blocking(move || -> miette::Result<()> { - use virt::{connect::Connect, domain::Domain}; - let conn = Connect::open(Some(&uri)) - .map_err(|e| miette::miette!("libvirt connect failed: {e}"))?; - if let Ok(dom) = Domain::lookup_by_name(&conn, &id_for_task) { - let _ = dom.undefine(); - } - Ok(()) - }) - .await - .into_diagnostic()??; - // Cleanup files - if let Some(p) = vm.overlay_path.as_ref() { - let _ = tokio::fs::remove_file(p).await; - } - if let Some(p) = vm.seed_iso_path.as_ref() { - let _ = tokio::fs::remove_file(p).await; - } - let _ = tokio::fs::remove_dir_all(&vm.work_dir).await; - info!(domain = %id, "libvirt destroyed"); - Ok(()) - } - - async fn state(&self, vm: &VmHandle) -> Result { - let id = vm.id.clone(); - let uri = self.uri.clone(); - let active = tokio::task::spawn_blocking(move || -> miette::Result { - use virt::{connect::Connect, domain::Domain}; - let conn = Connect::open(Some(&uri)) - .map_err(|e| miette::miette!("libvirt connect failed: {e}"))?; - let dom = Domain::lookup_by_name(&conn, &id) - .map_err(|e| miette::miette!("lookup domain failed: {e}"))?; - let active = dom.is_active().unwrap_or(false); - Ok(active) - }) - .await - .into_diagnostic()??; - Ok(if active { - VmState::Running - } else { - VmState::Stopped - }) - } -} - -#[cfg(target_os = "illumos")] -pub struct ZonesHypervisor; - -#[cfg(target_os = "illumos")] -#[async_trait] -impl Hypervisor for ZonesHypervisor { - async fn prepare(&self, spec: &VmSpec, ctx: &JobContext) -> Result { - use std::process::Command; - let id = format!("zone-{}", ctx.request_id); - // Create working directory under /var/lib/solstice-ci if possible - let work_dir = { - let base = std::path::Path::new("/var/lib/solstice-ci"); - let dir = if base.exists() && base.is_dir() && std::fs::metadata(base).is_ok() { - base.join(&id) - } else { - std::env::temp_dir().join("solstice-zones").join(&id) - }; - let _ = std::fs::create_dir_all(&dir); - #[cfg(unix)] - let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)); - dir - }; - - // Detect base image format - let base = spec.image_path.clone(); - let base_fmt = tokio::task::spawn_blocking(move || -> miette::Result { - let out = Command::new("qemu-img") - .args(["info", "--output=json"]) - .arg(&base) - .output() - .map_err(|e| miette::miette!("qemu-img not found or failed: {e}"))?; - if !out.status.success() { - return Err(miette::miette!( - "qemu-img info failed: {}", - String::from_utf8_lossy(&out.stderr) - )); - } - let v: serde_json::Value = serde_json::from_slice(&out.stdout) - .map_err(|e| miette::miette!("parse qemu-img info json failed: {e}"))?; - Ok(v.get("format") - .and_then(|f| f.as_str()) - .unwrap_or("raw") - .to_string()) - }) - .await - .into_diagnostic()??; - - // Ensure raw image for bhyve: convert if needed - let raw_path = if base_fmt != "raw" { - let out_path = work_dir.join("disk.raw"); - let src = spec.image_path.clone(); - let dst = out_path.clone(); - tokio::task::spawn_blocking(move || -> miette::Result<()> { - let out = Command::new("qemu-img") - .args(["convert", "-O", "raw"]) - .arg(&src) - .arg(&dst) - .output() - .map_err(|e| miette::miette!("qemu-img convert failed to start: {e}"))?; - if !out.status.success() { - return Err(miette::miette!( - "qemu-img convert failed: {}", - String::from_utf8_lossy(&out.stderr) - )); - } - Ok(()) - }) - .await - .into_diagnostic()??; - info!(label = %spec.label, src = ?spec.image_path, out = ?out_path, "converted image to raw for bhyve"); - out_path - } else { - spec.image_path.clone() - }; - - // Seed ISO creation left to future; for now, return handle with path in overlay_path - Ok(VmHandle { - id, - backend: BackendTag::Zones, - work_dir, - overlay_path: Some(raw_path), - seed_iso_path: None, - console_socket: None, - ssh_host_port: None, - mac_addr: None, - }) - } - async fn start(&self, _vm: &VmHandle) -> Result<()> { - Ok(()) - } - async fn stop(&self, _vm: &VmHandle, _t: Duration) -> Result<()> { - Ok(()) - } - async fn destroy(&self, _vm: VmHandle) -> Result<()> { - Ok(()) - } - async fn suspend(&self, _vm: &VmHandle) -> Result<()> { - Ok(()) - } -} diff --git a/crates/orchestrator/src/main.rs b/crates/orchestrator/src/main.rs index 5fcfc96..1c60423 100644 --- a/crates/orchestrator/src/main.rs +++ b/crates/orchestrator/src/main.rs @@ -17,7 +17,7 @@ use tracing::{debug, info, warn}; use crate::error::OrchestratorError; use crate::persist::{JobState, Persist}; use config::OrchestratorConfig; -use hypervisor::{JobContext, RouterHypervisor, VmSpec}; +use hypervisor::{JobContext, VmSpec}; use scheduler::{ExecConfig, SchedItem, Scheduler}; use std::sync::Arc; use tokio::sync::Notify; @@ -73,13 +73,10 @@ struct Opts { #[arg(long, env = "AMQP_PREFETCH")] amqp_prefetch: Option, - /// Libvirt URI (Linux only) - #[arg(long, env = "LIBVIRT_URI", default_value = "qemu:///system")] - libvirt_uri: String, - - /// Libvirt network name (Linux only) - #[arg(long, env = "LIBVIRT_NETWORK", default_value = "default")] - libvirt_network: String, + /// Network bridge name for TAP networking (e.g., "virbr0"). If empty, uses + /// user-mode (SLIRP) networking with SSH port forwarding — recommended for containers. + #[arg(long, env = "NETWORK_BRIDGE")] + network_bridge: Option, /// OTLP endpoint (e.g., http://localhost:4317) #[arg(long, env = "OTEL_EXPORTER_OTLP_ENDPOINT")] @@ -188,8 +185,8 @@ async fn main() -> Result<()> { // Capacity settings let capacity_map = parse_capacity_map(opts.capacity_map.as_deref()); - // Build hypervisor router - let router = RouterHypervisor::build(opts.libvirt_uri.clone(), opts.libvirt_network.clone()); + // Build hypervisor via vm-manager adapter (QEMU direct, no libvirt) + let router = vm_adapter::VmManagerAdapter::build(opts.network_bridge.clone()); // Initialize persistence (optional). Skip when requested for faster startup. let persist = if opts.skip_persistence { @@ -223,8 +220,6 @@ async fn main() -> Result<()> { runner_illumos_path: opts.runner_illumos_path.clone(), ssh_connect_timeout_secs: opts.ssh_connect_timeout_secs, boot_wait_secs: opts.boot_wait_secs, - libvirt_uri: opts.libvirt_uri.clone(), - libvirt_network: opts.libvirt_network.clone(), }; let sched = Scheduler::new( @@ -357,7 +352,7 @@ async fn main() -> Result<()> { cpu, ram_mb, disk_gb, - network: None, // libvirt network handled in backend + network: None, // vm-manager handles networking nocloud: image.nocloud, user_data: Some(make_cloud_init_userdata( &job.repo_url, diff --git a/crates/orchestrator/src/scheduler.rs b/crates/orchestrator/src/scheduler.rs index 68678fb..739fc06 100644 --- a/crates/orchestrator/src/scheduler.rs +++ b/crates/orchestrator/src/scheduler.rs @@ -40,8 +40,6 @@ pub struct ExecConfig { pub runner_illumos_path: String, pub ssh_connect_timeout_secs: u64, pub boot_wait_secs: u64, - pub libvirt_uri: String, - pub libvirt_network: String, } // Strip ANSI escape sequences (colors/control) from a string for clean log storage.