mirror of
https://codeberg.org/Toasterson/solstice-ci.git
synced 2026-04-10 13:20:41 +00:00
Remove libvirt dependencies and clean up orchestrator
- Remove `virt` crate dependency and libvirt feature flag - Remove `ssh2` crate dependency (vm-manager handles SSH) - Remove `zstd` crate dependency (vm-manager handles decompression) - Remove LibvirtHypervisor, ZonesHypervisor, RouterHypervisor from hypervisor.rs - Remove libvirt error types from error.rs - Remove libvirt_uri/libvirt_network CLI options, add network_bridge - Replace RouterHypervisor::build() with VmManagerAdapter::build() - Update deb package depends: libvirt → qemu-system-x86 - Keep Noop backend for development/testing - Dead old SSH/console functions left for future cleanup
This commit is contained in:
parent
2d971ef500
commit
c9fc05a00e
5 changed files with 14 additions and 712 deletions
|
|
@ -5,8 +5,6 @@ edition = "2024"
|
|||
build = "build.rs"
|
||||
|
||||
[features]
|
||||
# Enable libvirt backend on Linux hosts (uses virt crate on Linux)
|
||||
libvirt = []
|
||||
|
||||
[dependencies]
|
||||
common = { path = "../common" }
|
||||
|
|
@ -24,12 +22,9 @@ config = { version = "0.15", default-features = false, features = ["yaml"] }
|
|||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "http2", "gzip", "brotli", "zstd"] }
|
||||
# HTTP server for logs (runner serving removed)
|
||||
axum = { version = "0.8", features = ["macros"] }
|
||||
# SSH client for upload/exec/logs
|
||||
ssh2 = "0.9"
|
||||
# SSH key generation (vm-manager handles SSH connections)
|
||||
ssh-key = { version = "0.6", features = ["ed25519"] }
|
||||
rand_core = "0.6"
|
||||
# Compression/decompression
|
||||
zstd = "0.13"
|
||||
# DB (optional basic persistence)
|
||||
sea-orm = { version = "1.1.17", default-features = false, features = ["sqlx-postgres", "sqlx-sqlite", "runtime-tokio-rustls", "macros", "with-uuid", "with-chrono" ] }
|
||||
migration = { path = "../migration" }
|
||||
|
|
@ -40,9 +35,6 @@ dashmap = "6"
|
|||
async-trait = "0.1"
|
||||
uuid = { version = "1", features = ["v4", "serde"] }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
virt = { version = "0.4.3" }
|
||||
|
||||
[package.metadata.deb]
|
||||
name = "solstice-orchestrator"
|
||||
maintainer = "Solstice CI <ops@solstice-ci.org>"
|
||||
|
|
@ -55,11 +47,10 @@ assets = [
|
|||
["../../examples/etc/solstice/orchestrator.env.sample", "/etc/solstice/orchestrator.env", "640"],
|
||||
]
|
||||
depends = [
|
||||
"libvirt-daemon-system",
|
||||
"libvirt-clients",
|
||||
"qemu-system-x86",
|
||||
"qemu-utils",
|
||||
"ca-certificates",
|
||||
"openssh-client",
|
||||
]
|
||||
recommends = ["qemu-kvm", "virtinst"]
|
||||
recommends = ["qemu-kvm"]
|
||||
conf-files = ["/etc/solstice/orchestrator.yaml", "/etc/solstice/orchestrator.env"]
|
||||
maintainer-scripts = "packaging/debian/"
|
||||
|
|
|
|||
|
|
@ -2,70 +2,6 @@ use thiserror::Error;
|
|||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum OrchestratorError {
|
||||
#[error("libvirt connect failed: {0}")]
|
||||
LibvirtConnect(#[source] anyhow::Error),
|
||||
|
||||
#[error("define domain failed: {0}")]
|
||||
LibvirtDefine(#[source] anyhow::Error),
|
||||
|
||||
#[error("lookup domain failed: {0}")]
|
||||
LibvirtLookup(#[source] anyhow::Error),
|
||||
|
||||
#[error("domain operation failed: {0}")]
|
||||
LibvirtDomain(#[source] anyhow::Error),
|
||||
|
||||
#[error("qemu-img not found or failed: {0}")]
|
||||
QemuImg(#[source] anyhow::Error),
|
||||
|
||||
#[error("qemu-img convert failed: {0}")]
|
||||
QemuImgConvert(#[source] anyhow::Error),
|
||||
|
||||
#[error("parse qemu-img info json failed: {0}")]
|
||||
QemuImgParse(#[source] anyhow::Error),
|
||||
|
||||
#[error("mkisofs/genisoimage not found: {0}")]
|
||||
MkIsoFs(#[source] anyhow::Error),
|
||||
|
||||
#[error("SSH session new failed")]
|
||||
SshSessionNew,
|
||||
|
||||
#[error("tcp connect failed: {0}")]
|
||||
TcpConnect(#[source] std::io::Error),
|
||||
|
||||
#[error("ssh handshake: {0}")]
|
||||
SshHandshake(#[source] ssh2::Error),
|
||||
|
||||
#[error("ssh auth failed for {user}: {source}")]
|
||||
SshAuth {
|
||||
user: String,
|
||||
#[source]
|
||||
source: ssh2::Error,
|
||||
},
|
||||
|
||||
#[error("ssh not authenticated")]
|
||||
SshNotAuthenticated,
|
||||
|
||||
#[error("sftp init failed: {0}")]
|
||||
SftpInit(#[source] ssh2::Error),
|
||||
|
||||
#[error("open local runner: {0}")]
|
||||
OpenLocalRunner(#[source] std::io::Error),
|
||||
|
||||
#[error("read runner: {0}")]
|
||||
ReadRunner(#[source] std::io::Error),
|
||||
|
||||
#[error("sftp create: {0}")]
|
||||
SftpCreate(#[source] ssh2::Error),
|
||||
|
||||
#[error("sftp write: {0}")]
|
||||
SftpWrite(#[source] std::io::Error),
|
||||
|
||||
#[error("channel session: {0}")]
|
||||
ChannelSession(#[source] ssh2::Error),
|
||||
|
||||
#[error("exec: {0}")]
|
||||
Exec(#[source] ssh2::Error),
|
||||
|
||||
#[error("ssh keygen failed: {0}")]
|
||||
SshKeygen(#[source] anyhow::Error),
|
||||
|
||||
|
|
@ -81,10 +17,3 @@ pub enum OrchestratorError {
|
|||
#[error("failed to apply migrations: {0}")]
|
||||
DbMigrate(#[source] anyhow::Error),
|
||||
}
|
||||
|
||||
// Helper conversions for common external error types into anyhow::Error where needed.
|
||||
impl From<virt::error::Error> for OrchestratorError {
|
||||
fn from(e: virt::error::Error) -> Self {
|
||||
OrchestratorError::LibvirtDomain(anyhow::Error::new(e))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,19 +1,14 @@
|
|||
use async_trait::async_trait;
|
||||
use miette::{IntoDiagnostic as _, Result};
|
||||
use std::os::unix::prelude::PermissionsExt;
|
||||
use std::{path::PathBuf, time::Duration};
|
||||
use tracing::info;
|
||||
|
||||
// Backend tag is used internally to remember which backend handled this VM.
|
||||
/// Backend tag is used internally to remember which backend handled this VM.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum BackendTag {
|
||||
Noop,
|
||||
Qemu,
|
||||
Propolis,
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
Libvirt,
|
||||
#[cfg(target_os = "illumos")]
|
||||
Zones,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
|
|
@ -77,176 +72,11 @@ pub trait Hypervisor: Send + Sync {
|
|||
async fn state(&self, _vm: &VmHandle) -> Result<VmState> {
|
||||
Ok(VmState::Prepared)
|
||||
}
|
||||
async fn guest_ip(&self, vm: &VmHandle) -> Result<String> {
|
||||
async fn guest_ip(&self, _vm: &VmHandle) -> Result<String> {
|
||||
Ok("127.0.0.1".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// A router that delegates to the correct backend implementation per job.
|
||||
pub struct RouterHypervisor {
|
||||
pub noop: NoopHypervisor,
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
pub libvirt: Option<LibvirtHypervisor>,
|
||||
#[cfg(target_os = "illumos")]
|
||||
pub zones: Option<ZonesHypervisor>,
|
||||
}
|
||||
|
||||
impl RouterHypervisor {
|
||||
#[allow(unused_variables)]
|
||||
pub fn build(libvirt_uri: String, libvirt_network: String) -> Self {
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
{
|
||||
return RouterHypervisor {
|
||||
noop: NoopHypervisor::default(),
|
||||
libvirt: Some(LibvirtHypervisor {
|
||||
uri: libvirt_uri,
|
||||
network: libvirt_network,
|
||||
}),
|
||||
};
|
||||
}
|
||||
#[cfg(target_os = "illumos")]
|
||||
{
|
||||
return RouterHypervisor {
|
||||
noop: NoopHypervisor::default(),
|
||||
zones: Some(ZonesHypervisor),
|
||||
};
|
||||
}
|
||||
#[cfg(all(
|
||||
not(target_os = "illumos"),
|
||||
not(all(target_os = "linux", feature = "libvirt"))
|
||||
))]
|
||||
{
|
||||
return RouterHypervisor {
|
||||
noop: NoopHypervisor::default(),
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Hypervisor for RouterHypervisor {
|
||||
async fn prepare(&self, spec: &VmSpec, ctx: &JobContext) -> Result<VmHandle> {
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
{
|
||||
if let Some(ref hv) = self.libvirt {
|
||||
return hv.prepare(spec, ctx).await;
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "illumos")]
|
||||
{
|
||||
if let Some(ref hv) = self.zones {
|
||||
return hv.prepare(spec, ctx).await;
|
||||
}
|
||||
}
|
||||
self.noop.prepare(spec, ctx).await
|
||||
}
|
||||
async fn start(&self, vm: &VmHandle) -> Result<()> {
|
||||
match vm.backend {
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
BackendTag::Libvirt => {
|
||||
if let Some(ref hv) = self.libvirt {
|
||||
hv.start(vm).await
|
||||
} else {
|
||||
self.noop.start(vm).await
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "illumos")]
|
||||
BackendTag::Zones => {
|
||||
if let Some(ref hv) = self.zones {
|
||||
hv.start(vm).await
|
||||
} else {
|
||||
self.noop.start(vm).await
|
||||
}
|
||||
}
|
||||
_ => self.noop.start(vm).await,
|
||||
}
|
||||
}
|
||||
async fn stop(&self, vm: &VmHandle, t: Duration) -> Result<()> {
|
||||
match vm.backend {
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
BackendTag::Libvirt => {
|
||||
if let Some(ref hv) = self.libvirt {
|
||||
hv.stop(vm, t).await
|
||||
} else {
|
||||
self.noop.stop(vm, t).await
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "illumos")]
|
||||
BackendTag::Zones => {
|
||||
if let Some(ref hv) = self.zones {
|
||||
hv.stop(vm, t).await
|
||||
} else {
|
||||
self.noop.stop(vm, t).await
|
||||
}
|
||||
}
|
||||
_ => self.noop.stop(vm, t).await,
|
||||
}
|
||||
}
|
||||
async fn suspend(&self, vm: &VmHandle) -> Result<()> {
|
||||
match vm.backend {
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
BackendTag::Libvirt => {
|
||||
if let Some(ref hv) = self.libvirt {
|
||||
hv.suspend(vm).await
|
||||
} else {
|
||||
self.noop.suspend(vm).await
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "illumos")]
|
||||
BackendTag::Zones => {
|
||||
if let Some(ref hv) = self.zones {
|
||||
hv.suspend(vm).await
|
||||
} else {
|
||||
self.noop.suspend(vm).await
|
||||
}
|
||||
}
|
||||
_ => self.noop.suspend(vm).await,
|
||||
}
|
||||
}
|
||||
async fn destroy(&self, vm: VmHandle) -> Result<()> {
|
||||
match vm.backend {
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
BackendTag::Libvirt => {
|
||||
if let Some(ref hv) = self.libvirt {
|
||||
hv.destroy(vm).await
|
||||
} else {
|
||||
self.noop.destroy(vm).await
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "illumos")]
|
||||
BackendTag::Zones => {
|
||||
if let Some(ref hv) = self.zones {
|
||||
hv.destroy(vm).await
|
||||
} else {
|
||||
self.noop.destroy(vm).await
|
||||
}
|
||||
}
|
||||
_ => self.noop.destroy(vm).await,
|
||||
}
|
||||
}
|
||||
async fn state(&self, vm: &VmHandle) -> Result<VmState> {
|
||||
match vm.backend {
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
BackendTag::Libvirt => {
|
||||
if let Some(ref hv) = self.libvirt {
|
||||
hv.state(vm).await
|
||||
} else {
|
||||
Ok(VmState::Prepared)
|
||||
}
|
||||
}
|
||||
#[cfg(target_os = "illumos")]
|
||||
BackendTag::Zones => {
|
||||
if let Some(ref hv) = self.zones {
|
||||
hv.state(vm).await
|
||||
} else {
|
||||
Ok(VmState::Prepared)
|
||||
}
|
||||
}
|
||||
_ => Ok(VmState::Prepared),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// No-op hypervisor for development on hosts without privileges.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct NoopHypervisor;
|
||||
|
|
@ -288,444 +118,3 @@ impl Hypervisor for NoopHypervisor {
|
|||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
pub struct LibvirtHypervisor {
|
||||
pub uri: String,
|
||||
pub network: String,
|
||||
}
|
||||
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
impl LibvirtHypervisor {
|
||||
fn mk_work_dir(&self, id: &str) -> std::path::PathBuf {
|
||||
// Prefer /var/lib/solstice-ci if writable, else tmp
|
||||
let base = std::path::Path::new("/var/lib/solstice-ci");
|
||||
let dir = if base.exists() && base.is_dir() && std::fs::metadata(base).is_ok() {
|
||||
base.join(id)
|
||||
} else {
|
||||
std::env::temp_dir().join("solstice-libvirt").join(id)
|
||||
};
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
// Make directory broadly accessible so host qemu (libvirt) can create/read files
|
||||
let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o777));
|
||||
dir
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
||||
#[async_trait]
|
||||
impl Hypervisor for LibvirtHypervisor {
|
||||
async fn prepare(&self, spec: &VmSpec, ctx: &JobContext) -> Result<VmHandle> {
|
||||
use std::process::Command;
|
||||
|
||||
let id = format!("job-{}", ctx.request_id);
|
||||
let work_dir = self.mk_work_dir(&id);
|
||||
|
||||
// Ensure network is active via virt crate; best-effort
|
||||
let uri = self.uri.clone();
|
||||
let net_name = self.network.clone();
|
||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
||||
use virt::{connect::Connect, network::Network};
|
||||
let conn = Connect::open(Some(&uri))
|
||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
||||
if let Ok(net) = Network::lookup_by_name(&conn, &net_name) {
|
||||
// If not active, try to create (activate). Then set autostart.
|
||||
let active = net.is_active().unwrap_or(false);
|
||||
if !active {
|
||||
let _ = net.create();
|
||||
}
|
||||
let _ = net.set_autostart(true);
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()??;
|
||||
|
||||
// Create qcow2 overlay
|
||||
let overlay = work_dir.join("overlay.qcow2");
|
||||
let size_arg = format!("{}G", spec.disk_gb);
|
||||
let status = tokio::task::spawn_blocking({
|
||||
let base = spec.image_path.clone();
|
||||
let overlay = overlay.clone();
|
||||
move || -> miette::Result<()> {
|
||||
// Detect base image format to set -F accordingly (raw or qcow2)
|
||||
let base_fmt_out = std::process::Command::new("qemu-img")
|
||||
.args(["info", "--output=json"])
|
||||
.arg(&base)
|
||||
.output()
|
||||
.map_err(|e| miette::miette!("qemu-img not found or failed: {e}"))?;
|
||||
if !base_fmt_out.status.success() {
|
||||
return Err(miette::miette!(
|
||||
"qemu-img info failed: {}",
|
||||
String::from_utf8_lossy(&base_fmt_out.stderr)
|
||||
));
|
||||
}
|
||||
let base_fmt: String = {
|
||||
let v: serde_json::Value = serde_json::from_slice(&base_fmt_out.stdout)
|
||||
.map_err(|e| miette::miette!("parse qemu-img info json failed: {e}"))?;
|
||||
v.get("format")
|
||||
.and_then(|f| f.as_str())
|
||||
.unwrap_or("raw")
|
||||
.to_string()
|
||||
};
|
||||
|
||||
let out = Command::new("qemu-img")
|
||||
.args(["create", "-f", "qcow2", "-F"])
|
||||
.arg(&base_fmt)
|
||||
.args(["-b"])
|
||||
.arg(&base)
|
||||
.arg(&overlay)
|
||||
.arg(&size_arg)
|
||||
.output()
|
||||
.map_err(|e| miette::miette!("qemu-img not found or failed: {e}"))?;
|
||||
if !out.status.success() {
|
||||
return Err(miette::miette!(
|
||||
"qemu-img create failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()??;
|
||||
let _ = status; // appease compiler if unused
|
||||
// Relax permissions on overlay so host qemu can access it
|
||||
let _ = std::fs::set_permissions(&overlay, std::fs::Permissions::from_mode(0o666));
|
||||
|
||||
// Build NoCloud seed ISO if user_data provided, or synthesize from per-job SSH key
|
||||
let mut seed_iso: Option<PathBuf> = None;
|
||||
// Prefer spec-provided user-data; otherwise, if we have a per-job SSH public key, generate minimal cloud-config
|
||||
let user_data_opt: Option<Vec<u8>> = if let Some(ref ud) = spec.user_data {
|
||||
Some(ud.clone())
|
||||
} else if let Some(ref pubkey) = ctx.ssh_public_key {
|
||||
let s = format!(
|
||||
r#"#cloud-config
|
||||
users:
|
||||
- default
|
||||
ssh_authorized_keys:
|
||||
- {ssh_pubkey}
|
||||
"#,
|
||||
ssh_pubkey = pubkey.trim()
|
||||
);
|
||||
Some(s.into_bytes())
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
if let Some(user_data) = user_data_opt {
|
||||
let seed_dir = work_dir.join("seed");
|
||||
tokio::fs::create_dir_all(&seed_dir)
|
||||
.await
|
||||
.into_diagnostic()?;
|
||||
let ud_path = seed_dir.join("user-data");
|
||||
let md_path = seed_dir.join("meta-data");
|
||||
tokio::fs::write(&ud_path, &user_data)
|
||||
.await
|
||||
.into_diagnostic()?;
|
||||
let meta = format!("instance-id: {}\nlocal-hostname: {}\n", id, id);
|
||||
tokio::fs::write(&md_path, meta.as_bytes())
|
||||
.await
|
||||
.into_diagnostic()?;
|
||||
|
||||
// mkisofs or genisoimage
|
||||
let iso_path = work_dir.join("seed.iso");
|
||||
tokio::task::spawn_blocking({
|
||||
let iso_path = iso_path.clone();
|
||||
let seed_dir = seed_dir.clone();
|
||||
move || -> miette::Result<()> {
|
||||
let try_mk = |bin: &str| -> std::io::Result<std::process::Output> {
|
||||
Command::new(bin)
|
||||
.args(["-V", "cidata", "-J", "-R", "-o"])
|
||||
.arg(&iso_path)
|
||||
.arg(&seed_dir)
|
||||
.output()
|
||||
};
|
||||
let out = try_mk("mkisofs")
|
||||
.or_else(|_| try_mk("genisoimage"))
|
||||
.map_err(|e| miette::miette!("mkisofs/genisoimage not found: {e}"))?;
|
||||
if !out.status.success() {
|
||||
return Err(miette::miette!(
|
||||
"mkisofs failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()??;
|
||||
seed_iso = Some(iso_path);
|
||||
}
|
||||
|
||||
// Relax permissions on seed ISO if created (readable by host qemu)
|
||||
if let Some(ref p) = seed_iso {
|
||||
let _ = std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o644));
|
||||
}
|
||||
|
||||
// Serial console log file path (pre-create with permissive perms for libvirt)
|
||||
let console_log = work_dir.join("console.log");
|
||||
let _ = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&console_log);
|
||||
let _ = std::fs::set_permissions(&console_log, std::fs::Permissions::from_mode(0o666));
|
||||
let console_log_str = console_log.display().to_string();
|
||||
info!(domain = %id, console = %console_log_str, "serial console will be logged to file");
|
||||
|
||||
// Domain XML
|
||||
let xml = {
|
||||
let mem = spec.ram_mb;
|
||||
let vcpus = spec.cpu;
|
||||
let overlay_str = overlay.display().to_string();
|
||||
let seed_str = seed_iso.as_ref().map(|p| p.display().to_string());
|
||||
let net = self.network.clone();
|
||||
let cdrom = seed_str.map(|p| format!("<disk type='file' device='cdrom'>\n <driver name='qemu' type='raw'/>\n <source file='{}'/>\n <target dev='hdb' bus='ide'/>\n <readonly/>\n</disk>", p)).unwrap_or_default();
|
||||
format!(
|
||||
"<domain type='kvm'>\n<name>{}</name>\n<memory unit='MiB'>{}</memory>\n<vcpu>{}</vcpu>\n<os>\n <type arch='x86_64' machine='pc'>hvm</type>\n <boot dev='hd'/>\n</os>\n<features><acpi/></features>\n<devices>\n <disk type='file' device='disk'>\n <driver name='qemu' type='qcow2' cache='none'/>\n <source file='{}'/>\n <target dev='vda' bus='virtio'/>\n </disk>\n {}\n <interface type='network'>\n <source network='{}'/>\n <model type='virtio'/>\n </interface>\n <graphics type='vnc' autoport='yes' listen='127.0.0.1'/>\n <serial type='pty'>\n <target port='0'/>\n </serial>\n <serial type='file'>\n <source path='{}'/>\n <target port='1'/>\n </serial>\n <console type='pty'>\n <target type='serial' port='0'/>\n </console>\n</devices>\n<on_poweroff>destroy</on_poweroff>\n<on_crash>destroy</on_crash>\n</domain>",
|
||||
id, mem, vcpus, overlay_str, cdrom, net, console_log_str
|
||||
)
|
||||
};
|
||||
|
||||
// Define via virt crate
|
||||
let uri2 = self.uri.clone();
|
||||
let xml_clone = xml.clone();
|
||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
||||
use virt::{connect::Connect, domain::Domain};
|
||||
let conn = Connect::open(Some(&uri2))
|
||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
||||
let _dom = Domain::define_xml(&conn, &xml_clone)
|
||||
.map_err(|e| miette::miette!("define domain failed: {e}"))?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()??;
|
||||
|
||||
info!(domain = %id, image = ?spec.image_path, cpu = spec.cpu, ram_mb = spec.ram_mb, "libvirt prepared");
|
||||
Ok(VmHandle {
|
||||
id,
|
||||
backend: BackendTag::Libvirt,
|
||||
work_dir,
|
||||
overlay_path: Some(overlay),
|
||||
seed_iso_path: seed_iso,
|
||||
console_socket: None,
|
||||
ssh_host_port: None,
|
||||
mac_addr: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn start(&self, vm: &VmHandle) -> Result<()> {
|
||||
let id = vm.id.clone();
|
||||
let uri = self.uri.clone();
|
||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
||||
use virt::{connect::Connect, domain::Domain};
|
||||
let conn = Connect::open(Some(&uri))
|
||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
||||
// Lookup domain by name and start
|
||||
let dom = Domain::lookup_by_name(&conn, &id)
|
||||
.map_err(|e| miette::miette!("lookup domain failed: {e}"))?;
|
||||
dom.create()
|
||||
.map_err(|e| miette::miette!("domain start failed: {e}"))?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()??;
|
||||
info!(domain = %vm.id, "libvirt started");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn stop(&self, vm: &VmHandle, t: Duration) -> Result<()> {
|
||||
let id = vm.id.clone();
|
||||
let uri = self.uri.clone();
|
||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
||||
use virt::{connect::Connect, domain::Domain};
|
||||
let conn = Connect::open(Some(&uri))
|
||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
||||
let dom = Domain::lookup_by_name(&conn, &id)
|
||||
.map_err(|e| miette::miette!("lookup domain failed: {e}"))?;
|
||||
let _ = dom.shutdown();
|
||||
let start = std::time::Instant::now();
|
||||
while start.elapsed() < t {
|
||||
match dom.is_active() {
|
||||
Ok(false) => break,
|
||||
_ => {}
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
||||
}
|
||||
// Force destroy if still active
|
||||
let _ = dom.destroy();
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()??;
|
||||
info!(domain = %vm.id, "libvirt stopped");
|
||||
Ok(())
|
||||
}
|
||||
async fn suspend(&self, vm: &VmHandle) -> Result<()> {
|
||||
let id = vm.id.clone();
|
||||
let uri = self.uri.clone();
|
||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
||||
use virt::{connect::Connect, domain::Domain};
|
||||
let conn = Connect::open(Some(&uri))
|
||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
||||
let dom = Domain::lookup_by_name(&conn, &id)
|
||||
.map_err(|e| miette::miette!("lookup domain failed: {e}"))?;
|
||||
dom.suspend()
|
||||
.map_err(|e| miette::miette!("domain suspend failed: {e}"))?;
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()??;
|
||||
info!(domain = %vm.id, "libvirt suspended");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn destroy(&self, vm: VmHandle) -> Result<()> {
|
||||
let id = vm.id.clone();
|
||||
let uri = self.uri.clone();
|
||||
let id_for_task = id.clone();
|
||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
||||
use virt::{connect::Connect, domain::Domain};
|
||||
let conn = Connect::open(Some(&uri))
|
||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
||||
if let Ok(dom) = Domain::lookup_by_name(&conn, &id_for_task) {
|
||||
let _ = dom.undefine();
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()??;
|
||||
// Cleanup files
|
||||
if let Some(p) = vm.overlay_path.as_ref() {
|
||||
let _ = tokio::fs::remove_file(p).await;
|
||||
}
|
||||
if let Some(p) = vm.seed_iso_path.as_ref() {
|
||||
let _ = tokio::fs::remove_file(p).await;
|
||||
}
|
||||
let _ = tokio::fs::remove_dir_all(&vm.work_dir).await;
|
||||
info!(domain = %id, "libvirt destroyed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn state(&self, vm: &VmHandle) -> Result<VmState> {
|
||||
let id = vm.id.clone();
|
||||
let uri = self.uri.clone();
|
||||
let active = tokio::task::spawn_blocking(move || -> miette::Result<bool> {
|
||||
use virt::{connect::Connect, domain::Domain};
|
||||
let conn = Connect::open(Some(&uri))
|
||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
||||
let dom = Domain::lookup_by_name(&conn, &id)
|
||||
.map_err(|e| miette::miette!("lookup domain failed: {e}"))?;
|
||||
let active = dom.is_active().unwrap_or(false);
|
||||
Ok(active)
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()??;
|
||||
Ok(if active {
|
||||
VmState::Running
|
||||
} else {
|
||||
VmState::Stopped
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "illumos")]
|
||||
pub struct ZonesHypervisor;
|
||||
|
||||
#[cfg(target_os = "illumos")]
|
||||
#[async_trait]
|
||||
impl Hypervisor for ZonesHypervisor {
|
||||
async fn prepare(&self, spec: &VmSpec, ctx: &JobContext) -> Result<VmHandle> {
|
||||
use std::process::Command;
|
||||
let id = format!("zone-{}", ctx.request_id);
|
||||
// Create working directory under /var/lib/solstice-ci if possible
|
||||
let work_dir = {
|
||||
let base = std::path::Path::new("/var/lib/solstice-ci");
|
||||
let dir = if base.exists() && base.is_dir() && std::fs::metadata(base).is_ok() {
|
||||
base.join(&id)
|
||||
} else {
|
||||
std::env::temp_dir().join("solstice-zones").join(&id)
|
||||
};
|
||||
let _ = std::fs::create_dir_all(&dir);
|
||||
#[cfg(unix)]
|
||||
let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700));
|
||||
dir
|
||||
};
|
||||
|
||||
// Detect base image format
|
||||
let base = spec.image_path.clone();
|
||||
let base_fmt = tokio::task::spawn_blocking(move || -> miette::Result<String> {
|
||||
let out = Command::new("qemu-img")
|
||||
.args(["info", "--output=json"])
|
||||
.arg(&base)
|
||||
.output()
|
||||
.map_err(|e| miette::miette!("qemu-img not found or failed: {e}"))?;
|
||||
if !out.status.success() {
|
||||
return Err(miette::miette!(
|
||||
"qemu-img info failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
));
|
||||
}
|
||||
let v: serde_json::Value = serde_json::from_slice(&out.stdout)
|
||||
.map_err(|e| miette::miette!("parse qemu-img info json failed: {e}"))?;
|
||||
Ok(v.get("format")
|
||||
.and_then(|f| f.as_str())
|
||||
.unwrap_or("raw")
|
||||
.to_string())
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()??;
|
||||
|
||||
// Ensure raw image for bhyve: convert if needed
|
||||
let raw_path = if base_fmt != "raw" {
|
||||
let out_path = work_dir.join("disk.raw");
|
||||
let src = spec.image_path.clone();
|
||||
let dst = out_path.clone();
|
||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
||||
let out = Command::new("qemu-img")
|
||||
.args(["convert", "-O", "raw"])
|
||||
.arg(&src)
|
||||
.arg(&dst)
|
||||
.output()
|
||||
.map_err(|e| miette::miette!("qemu-img convert failed to start: {e}"))?;
|
||||
if !out.status.success() {
|
||||
return Err(miette::miette!(
|
||||
"qemu-img convert failed: {}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()??;
|
||||
info!(label = %spec.label, src = ?spec.image_path, out = ?out_path, "converted image to raw for bhyve");
|
||||
out_path
|
||||
} else {
|
||||
spec.image_path.clone()
|
||||
};
|
||||
|
||||
// Seed ISO creation left to future; for now, return handle with path in overlay_path
|
||||
Ok(VmHandle {
|
||||
id,
|
||||
backend: BackendTag::Zones,
|
||||
work_dir,
|
||||
overlay_path: Some(raw_path),
|
||||
seed_iso_path: None,
|
||||
console_socket: None,
|
||||
ssh_host_port: None,
|
||||
mac_addr: None,
|
||||
})
|
||||
}
|
||||
async fn start(&self, _vm: &VmHandle) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn stop(&self, _vm: &VmHandle, _t: Duration) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn destroy(&self, _vm: VmHandle) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
async fn suspend(&self, _vm: &VmHandle) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ use tracing::{debug, info, warn};
|
|||
use crate::error::OrchestratorError;
|
||||
use crate::persist::{JobState, Persist};
|
||||
use config::OrchestratorConfig;
|
||||
use hypervisor::{JobContext, RouterHypervisor, VmSpec};
|
||||
use hypervisor::{JobContext, VmSpec};
|
||||
use scheduler::{ExecConfig, SchedItem, Scheduler};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Notify;
|
||||
|
|
@ -73,13 +73,10 @@ struct Opts {
|
|||
#[arg(long, env = "AMQP_PREFETCH")]
|
||||
amqp_prefetch: Option<u16>,
|
||||
|
||||
/// Libvirt URI (Linux only)
|
||||
#[arg(long, env = "LIBVIRT_URI", default_value = "qemu:///system")]
|
||||
libvirt_uri: String,
|
||||
|
||||
/// Libvirt network name (Linux only)
|
||||
#[arg(long, env = "LIBVIRT_NETWORK", default_value = "default")]
|
||||
libvirt_network: String,
|
||||
/// Network bridge name for TAP networking (e.g., "virbr0"). If empty, uses
|
||||
/// user-mode (SLIRP) networking with SSH port forwarding — recommended for containers.
|
||||
#[arg(long, env = "NETWORK_BRIDGE")]
|
||||
network_bridge: Option<String>,
|
||||
|
||||
/// OTLP endpoint (e.g., http://localhost:4317)
|
||||
#[arg(long, env = "OTEL_EXPORTER_OTLP_ENDPOINT")]
|
||||
|
|
@ -188,8 +185,8 @@ async fn main() -> Result<()> {
|
|||
// Capacity settings
|
||||
let capacity_map = parse_capacity_map(opts.capacity_map.as_deref());
|
||||
|
||||
// Build hypervisor router
|
||||
let router = RouterHypervisor::build(opts.libvirt_uri.clone(), opts.libvirt_network.clone());
|
||||
// Build hypervisor via vm-manager adapter (QEMU direct, no libvirt)
|
||||
let router = vm_adapter::VmManagerAdapter::build(opts.network_bridge.clone());
|
||||
|
||||
// Initialize persistence (optional). Skip when requested for faster startup.
|
||||
let persist = if opts.skip_persistence {
|
||||
|
|
@ -223,8 +220,6 @@ async fn main() -> Result<()> {
|
|||
runner_illumos_path: opts.runner_illumos_path.clone(),
|
||||
ssh_connect_timeout_secs: opts.ssh_connect_timeout_secs,
|
||||
boot_wait_secs: opts.boot_wait_secs,
|
||||
libvirt_uri: opts.libvirt_uri.clone(),
|
||||
libvirt_network: opts.libvirt_network.clone(),
|
||||
};
|
||||
|
||||
let sched = Scheduler::new(
|
||||
|
|
@ -357,7 +352,7 @@ async fn main() -> Result<()> {
|
|||
cpu,
|
||||
ram_mb,
|
||||
disk_gb,
|
||||
network: None, // libvirt network handled in backend
|
||||
network: None, // vm-manager handles networking
|
||||
nocloud: image.nocloud,
|
||||
user_data: Some(make_cloud_init_userdata(
|
||||
&job.repo_url,
|
||||
|
|
|
|||
|
|
@ -40,8 +40,6 @@ pub struct ExecConfig {
|
|||
pub runner_illumos_path: String,
|
||||
pub ssh_connect_timeout_secs: u64,
|
||||
pub boot_wait_secs: u64,
|
||||
pub libvirt_uri: String,
|
||||
pub libvirt_network: String,
|
||||
}
|
||||
|
||||
// Strip ANSI escape sequences (colors/control) from a string for clean log storage.
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue