mirror of
https://codeberg.org/Toasterson/solstice-ci.git
synced 2026-04-10 21:30:41 +00:00
Remove libvirt dependencies and clean up orchestrator
- Remove `virt` crate dependency and libvirt feature flag - Remove `ssh2` crate dependency (vm-manager handles SSH) - Remove `zstd` crate dependency (vm-manager handles decompression) - Remove LibvirtHypervisor, ZonesHypervisor, RouterHypervisor from hypervisor.rs - Remove libvirt error types from error.rs - Remove libvirt_uri/libvirt_network CLI options, add network_bridge - Replace RouterHypervisor::build() with VmManagerAdapter::build() - Update deb package depends: libvirt → qemu-system-x86 - Keep Noop backend for development/testing - Dead old SSH/console functions left for future cleanup
This commit is contained in:
parent
2d971ef500
commit
c9fc05a00e
5 changed files with 14 additions and 712 deletions
|
|
@ -5,8 +5,6 @@ edition = "2024"
|
||||||
build = "build.rs"
|
build = "build.rs"
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
# Enable libvirt backend on Linux hosts (uses virt crate on Linux)
|
|
||||||
libvirt = []
|
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
common = { path = "../common" }
|
common = { path = "../common" }
|
||||||
|
|
@ -24,12 +22,9 @@ config = { version = "0.15", default-features = false, features = ["yaml"] }
|
||||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "http2", "gzip", "brotli", "zstd"] }
|
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "http2", "gzip", "brotli", "zstd"] }
|
||||||
# HTTP server for logs (runner serving removed)
|
# HTTP server for logs (runner serving removed)
|
||||||
axum = { version = "0.8", features = ["macros"] }
|
axum = { version = "0.8", features = ["macros"] }
|
||||||
# SSH client for upload/exec/logs
|
# SSH key generation (vm-manager handles SSH connections)
|
||||||
ssh2 = "0.9"
|
|
||||||
ssh-key = { version = "0.6", features = ["ed25519"] }
|
ssh-key = { version = "0.6", features = ["ed25519"] }
|
||||||
rand_core = "0.6"
|
rand_core = "0.6"
|
||||||
# Compression/decompression
|
|
||||||
zstd = "0.13"
|
|
||||||
# DB (optional basic persistence)
|
# DB (optional basic persistence)
|
||||||
sea-orm = { version = "1.1.17", default-features = false, features = ["sqlx-postgres", "sqlx-sqlite", "runtime-tokio-rustls", "macros", "with-uuid", "with-chrono" ] }
|
sea-orm = { version = "1.1.17", default-features = false, features = ["sqlx-postgres", "sqlx-sqlite", "runtime-tokio-rustls", "macros", "with-uuid", "with-chrono" ] }
|
||||||
migration = { path = "../migration" }
|
migration = { path = "../migration" }
|
||||||
|
|
@ -40,9 +35,6 @@ dashmap = "6"
|
||||||
async-trait = "0.1"
|
async-trait = "0.1"
|
||||||
uuid = { version = "1", features = ["v4", "serde"] }
|
uuid = { version = "1", features = ["v4", "serde"] }
|
||||||
|
|
||||||
[target.'cfg(target_os = "linux")'.dependencies]
|
|
||||||
virt = { version = "0.4.3" }
|
|
||||||
|
|
||||||
[package.metadata.deb]
|
[package.metadata.deb]
|
||||||
name = "solstice-orchestrator"
|
name = "solstice-orchestrator"
|
||||||
maintainer = "Solstice CI <ops@solstice-ci.org>"
|
maintainer = "Solstice CI <ops@solstice-ci.org>"
|
||||||
|
|
@ -55,11 +47,10 @@ assets = [
|
||||||
["../../examples/etc/solstice/orchestrator.env.sample", "/etc/solstice/orchestrator.env", "640"],
|
["../../examples/etc/solstice/orchestrator.env.sample", "/etc/solstice/orchestrator.env", "640"],
|
||||||
]
|
]
|
||||||
depends = [
|
depends = [
|
||||||
"libvirt-daemon-system",
|
"qemu-system-x86",
|
||||||
"libvirt-clients",
|
"qemu-utils",
|
||||||
"ca-certificates",
|
"ca-certificates",
|
||||||
"openssh-client",
|
|
||||||
]
|
]
|
||||||
recommends = ["qemu-kvm", "virtinst"]
|
recommends = ["qemu-kvm"]
|
||||||
conf-files = ["/etc/solstice/orchestrator.yaml", "/etc/solstice/orchestrator.env"]
|
conf-files = ["/etc/solstice/orchestrator.yaml", "/etc/solstice/orchestrator.env"]
|
||||||
maintainer-scripts = "packaging/debian/"
|
maintainer-scripts = "packaging/debian/"
|
||||||
|
|
|
||||||
|
|
@ -2,70 +2,6 @@ use thiserror::Error;
|
||||||
|
|
||||||
#[derive(Debug, Error)]
|
#[derive(Debug, Error)]
|
||||||
pub enum OrchestratorError {
|
pub enum OrchestratorError {
|
||||||
#[error("libvirt connect failed: {0}")]
|
|
||||||
LibvirtConnect(#[source] anyhow::Error),
|
|
||||||
|
|
||||||
#[error("define domain failed: {0}")]
|
|
||||||
LibvirtDefine(#[source] anyhow::Error),
|
|
||||||
|
|
||||||
#[error("lookup domain failed: {0}")]
|
|
||||||
LibvirtLookup(#[source] anyhow::Error),
|
|
||||||
|
|
||||||
#[error("domain operation failed: {0}")]
|
|
||||||
LibvirtDomain(#[source] anyhow::Error),
|
|
||||||
|
|
||||||
#[error("qemu-img not found or failed: {0}")]
|
|
||||||
QemuImg(#[source] anyhow::Error),
|
|
||||||
|
|
||||||
#[error("qemu-img convert failed: {0}")]
|
|
||||||
QemuImgConvert(#[source] anyhow::Error),
|
|
||||||
|
|
||||||
#[error("parse qemu-img info json failed: {0}")]
|
|
||||||
QemuImgParse(#[source] anyhow::Error),
|
|
||||||
|
|
||||||
#[error("mkisofs/genisoimage not found: {0}")]
|
|
||||||
MkIsoFs(#[source] anyhow::Error),
|
|
||||||
|
|
||||||
#[error("SSH session new failed")]
|
|
||||||
SshSessionNew,
|
|
||||||
|
|
||||||
#[error("tcp connect failed: {0}")]
|
|
||||||
TcpConnect(#[source] std::io::Error),
|
|
||||||
|
|
||||||
#[error("ssh handshake: {0}")]
|
|
||||||
SshHandshake(#[source] ssh2::Error),
|
|
||||||
|
|
||||||
#[error("ssh auth failed for {user}: {source}")]
|
|
||||||
SshAuth {
|
|
||||||
user: String,
|
|
||||||
#[source]
|
|
||||||
source: ssh2::Error,
|
|
||||||
},
|
|
||||||
|
|
||||||
#[error("ssh not authenticated")]
|
|
||||||
SshNotAuthenticated,
|
|
||||||
|
|
||||||
#[error("sftp init failed: {0}")]
|
|
||||||
SftpInit(#[source] ssh2::Error),
|
|
||||||
|
|
||||||
#[error("open local runner: {0}")]
|
|
||||||
OpenLocalRunner(#[source] std::io::Error),
|
|
||||||
|
|
||||||
#[error("read runner: {0}")]
|
|
||||||
ReadRunner(#[source] std::io::Error),
|
|
||||||
|
|
||||||
#[error("sftp create: {0}")]
|
|
||||||
SftpCreate(#[source] ssh2::Error),
|
|
||||||
|
|
||||||
#[error("sftp write: {0}")]
|
|
||||||
SftpWrite(#[source] std::io::Error),
|
|
||||||
|
|
||||||
#[error("channel session: {0}")]
|
|
||||||
ChannelSession(#[source] ssh2::Error),
|
|
||||||
|
|
||||||
#[error("exec: {0}")]
|
|
||||||
Exec(#[source] ssh2::Error),
|
|
||||||
|
|
||||||
#[error("ssh keygen failed: {0}")]
|
#[error("ssh keygen failed: {0}")]
|
||||||
SshKeygen(#[source] anyhow::Error),
|
SshKeygen(#[source] anyhow::Error),
|
||||||
|
|
||||||
|
|
@ -81,10 +17,3 @@ pub enum OrchestratorError {
|
||||||
#[error("failed to apply migrations: {0}")]
|
#[error("failed to apply migrations: {0}")]
|
||||||
DbMigrate(#[source] anyhow::Error),
|
DbMigrate(#[source] anyhow::Error),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Helper conversions for common external error types into anyhow::Error where needed.
|
|
||||||
impl From<virt::error::Error> for OrchestratorError {
|
|
||||||
fn from(e: virt::error::Error) -> Self {
|
|
||||||
OrchestratorError::LibvirtDomain(anyhow::Error::new(e))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,19 +1,14 @@
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use miette::{IntoDiagnostic as _, Result};
|
use miette::{IntoDiagnostic as _, Result};
|
||||||
use std::os::unix::prelude::PermissionsExt;
|
|
||||||
use std::{path::PathBuf, time::Duration};
|
use std::{path::PathBuf, time::Duration};
|
||||||
use tracing::info;
|
use tracing::info;
|
||||||
|
|
||||||
// Backend tag is used internally to remember which backend handled this VM.
|
/// Backend tag is used internally to remember which backend handled this VM.
|
||||||
#[derive(Debug, Clone, Copy)]
|
#[derive(Debug, Clone, Copy)]
|
||||||
pub enum BackendTag {
|
pub enum BackendTag {
|
||||||
Noop,
|
Noop,
|
||||||
Qemu,
|
Qemu,
|
||||||
Propolis,
|
Propolis,
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
Libvirt,
|
|
||||||
#[cfg(target_os = "illumos")]
|
|
||||||
Zones,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
|
|
@ -77,176 +72,11 @@ pub trait Hypervisor: Send + Sync {
|
||||||
async fn state(&self, _vm: &VmHandle) -> Result<VmState> {
|
async fn state(&self, _vm: &VmHandle) -> Result<VmState> {
|
||||||
Ok(VmState::Prepared)
|
Ok(VmState::Prepared)
|
||||||
}
|
}
|
||||||
async fn guest_ip(&self, vm: &VmHandle) -> Result<String> {
|
async fn guest_ip(&self, _vm: &VmHandle) -> Result<String> {
|
||||||
Ok("127.0.0.1".to_string())
|
Ok("127.0.0.1".to_string())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A router that delegates to the correct backend implementation per job.
|
|
||||||
pub struct RouterHypervisor {
|
|
||||||
pub noop: NoopHypervisor,
|
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
pub libvirt: Option<LibvirtHypervisor>,
|
|
||||||
#[cfg(target_os = "illumos")]
|
|
||||||
pub zones: Option<ZonesHypervisor>,
|
|
||||||
}
|
|
||||||
|
|
||||||
impl RouterHypervisor {
|
|
||||||
#[allow(unused_variables)]
|
|
||||||
pub fn build(libvirt_uri: String, libvirt_network: String) -> Self {
|
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
{
|
|
||||||
return RouterHypervisor {
|
|
||||||
noop: NoopHypervisor::default(),
|
|
||||||
libvirt: Some(LibvirtHypervisor {
|
|
||||||
uri: libvirt_uri,
|
|
||||||
network: libvirt_network,
|
|
||||||
}),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
#[cfg(target_os = "illumos")]
|
|
||||||
{
|
|
||||||
return RouterHypervisor {
|
|
||||||
noop: NoopHypervisor::default(),
|
|
||||||
zones: Some(ZonesHypervisor),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
#[cfg(all(
|
|
||||||
not(target_os = "illumos"),
|
|
||||||
not(all(target_os = "linux", feature = "libvirt"))
|
|
||||||
))]
|
|
||||||
{
|
|
||||||
return RouterHypervisor {
|
|
||||||
noop: NoopHypervisor::default(),
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[async_trait]
|
|
||||||
impl Hypervisor for RouterHypervisor {
|
|
||||||
async fn prepare(&self, spec: &VmSpec, ctx: &JobContext) -> Result<VmHandle> {
|
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
{
|
|
||||||
if let Some(ref hv) = self.libvirt {
|
|
||||||
return hv.prepare(spec, ctx).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#[cfg(target_os = "illumos")]
|
|
||||||
{
|
|
||||||
if let Some(ref hv) = self.zones {
|
|
||||||
return hv.prepare(spec, ctx).await;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
self.noop.prepare(spec, ctx).await
|
|
||||||
}
|
|
||||||
async fn start(&self, vm: &VmHandle) -> Result<()> {
|
|
||||||
match vm.backend {
|
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
BackendTag::Libvirt => {
|
|
||||||
if let Some(ref hv) = self.libvirt {
|
|
||||||
hv.start(vm).await
|
|
||||||
} else {
|
|
||||||
self.noop.start(vm).await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#[cfg(target_os = "illumos")]
|
|
||||||
BackendTag::Zones => {
|
|
||||||
if let Some(ref hv) = self.zones {
|
|
||||||
hv.start(vm).await
|
|
||||||
} else {
|
|
||||||
self.noop.start(vm).await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => self.noop.start(vm).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
async fn stop(&self, vm: &VmHandle, t: Duration) -> Result<()> {
|
|
||||||
match vm.backend {
|
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
BackendTag::Libvirt => {
|
|
||||||
if let Some(ref hv) = self.libvirt {
|
|
||||||
hv.stop(vm, t).await
|
|
||||||
} else {
|
|
||||||
self.noop.stop(vm, t).await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#[cfg(target_os = "illumos")]
|
|
||||||
BackendTag::Zones => {
|
|
||||||
if let Some(ref hv) = self.zones {
|
|
||||||
hv.stop(vm, t).await
|
|
||||||
} else {
|
|
||||||
self.noop.stop(vm, t).await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => self.noop.stop(vm, t).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
async fn suspend(&self, vm: &VmHandle) -> Result<()> {
|
|
||||||
match vm.backend {
|
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
BackendTag::Libvirt => {
|
|
||||||
if let Some(ref hv) = self.libvirt {
|
|
||||||
hv.suspend(vm).await
|
|
||||||
} else {
|
|
||||||
self.noop.suspend(vm).await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#[cfg(target_os = "illumos")]
|
|
||||||
BackendTag::Zones => {
|
|
||||||
if let Some(ref hv) = self.zones {
|
|
||||||
hv.suspend(vm).await
|
|
||||||
} else {
|
|
||||||
self.noop.suspend(vm).await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => self.noop.suspend(vm).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
async fn destroy(&self, vm: VmHandle) -> Result<()> {
|
|
||||||
match vm.backend {
|
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
BackendTag::Libvirt => {
|
|
||||||
if let Some(ref hv) = self.libvirt {
|
|
||||||
hv.destroy(vm).await
|
|
||||||
} else {
|
|
||||||
self.noop.destroy(vm).await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#[cfg(target_os = "illumos")]
|
|
||||||
BackendTag::Zones => {
|
|
||||||
if let Some(ref hv) = self.zones {
|
|
||||||
hv.destroy(vm).await
|
|
||||||
} else {
|
|
||||||
self.noop.destroy(vm).await
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => self.noop.destroy(vm).await,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
async fn state(&self, vm: &VmHandle) -> Result<VmState> {
|
|
||||||
match vm.backend {
|
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
BackendTag::Libvirt => {
|
|
||||||
if let Some(ref hv) = self.libvirt {
|
|
||||||
hv.state(vm).await
|
|
||||||
} else {
|
|
||||||
Ok(VmState::Prepared)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
#[cfg(target_os = "illumos")]
|
|
||||||
BackendTag::Zones => {
|
|
||||||
if let Some(ref hv) = self.zones {
|
|
||||||
hv.state(vm).await
|
|
||||||
} else {
|
|
||||||
Ok(VmState::Prepared)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
_ => Ok(VmState::Prepared),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// No-op hypervisor for development on hosts without privileges.
|
/// No-op hypervisor for development on hosts without privileges.
|
||||||
#[derive(Debug, Clone, Default)]
|
#[derive(Debug, Clone, Default)]
|
||||||
pub struct NoopHypervisor;
|
pub struct NoopHypervisor;
|
||||||
|
|
@ -288,444 +118,3 @@ impl Hypervisor for NoopHypervisor {
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
pub struct LibvirtHypervisor {
|
|
||||||
pub uri: String,
|
|
||||||
pub network: String,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
impl LibvirtHypervisor {
|
|
||||||
fn mk_work_dir(&self, id: &str) -> std::path::PathBuf {
|
|
||||||
// Prefer /var/lib/solstice-ci if writable, else tmp
|
|
||||||
let base = std::path::Path::new("/var/lib/solstice-ci");
|
|
||||||
let dir = if base.exists() && base.is_dir() && std::fs::metadata(base).is_ok() {
|
|
||||||
base.join(id)
|
|
||||||
} else {
|
|
||||||
std::env::temp_dir().join("solstice-libvirt").join(id)
|
|
||||||
};
|
|
||||||
let _ = std::fs::create_dir_all(&dir);
|
|
||||||
// Make directory broadly accessible so host qemu (libvirt) can create/read files
|
|
||||||
let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o777));
|
|
||||||
dir
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(all(target_os = "linux", feature = "libvirt"))]
|
|
||||||
#[async_trait]
|
|
||||||
impl Hypervisor for LibvirtHypervisor {
|
|
||||||
async fn prepare(&self, spec: &VmSpec, ctx: &JobContext) -> Result<VmHandle> {
|
|
||||||
use std::process::Command;
|
|
||||||
|
|
||||||
let id = format!("job-{}", ctx.request_id);
|
|
||||||
let work_dir = self.mk_work_dir(&id);
|
|
||||||
|
|
||||||
// Ensure network is active via virt crate; best-effort
|
|
||||||
let uri = self.uri.clone();
|
|
||||||
let net_name = self.network.clone();
|
|
||||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
|
||||||
use virt::{connect::Connect, network::Network};
|
|
||||||
let conn = Connect::open(Some(&uri))
|
|
||||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
|
||||||
if let Ok(net) = Network::lookup_by_name(&conn, &net_name) {
|
|
||||||
// If not active, try to create (activate). Then set autostart.
|
|
||||||
let active = net.is_active().unwrap_or(false);
|
|
||||||
if !active {
|
|
||||||
let _ = net.create();
|
|
||||||
}
|
|
||||||
let _ = net.set_autostart(true);
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.into_diagnostic()??;
|
|
||||||
|
|
||||||
// Create qcow2 overlay
|
|
||||||
let overlay = work_dir.join("overlay.qcow2");
|
|
||||||
let size_arg = format!("{}G", spec.disk_gb);
|
|
||||||
let status = tokio::task::spawn_blocking({
|
|
||||||
let base = spec.image_path.clone();
|
|
||||||
let overlay = overlay.clone();
|
|
||||||
move || -> miette::Result<()> {
|
|
||||||
// Detect base image format to set -F accordingly (raw or qcow2)
|
|
||||||
let base_fmt_out = std::process::Command::new("qemu-img")
|
|
||||||
.args(["info", "--output=json"])
|
|
||||||
.arg(&base)
|
|
||||||
.output()
|
|
||||||
.map_err(|e| miette::miette!("qemu-img not found or failed: {e}"))?;
|
|
||||||
if !base_fmt_out.status.success() {
|
|
||||||
return Err(miette::miette!(
|
|
||||||
"qemu-img info failed: {}",
|
|
||||||
String::from_utf8_lossy(&base_fmt_out.stderr)
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let base_fmt: String = {
|
|
||||||
let v: serde_json::Value = serde_json::from_slice(&base_fmt_out.stdout)
|
|
||||||
.map_err(|e| miette::miette!("parse qemu-img info json failed: {e}"))?;
|
|
||||||
v.get("format")
|
|
||||||
.and_then(|f| f.as_str())
|
|
||||||
.unwrap_or("raw")
|
|
||||||
.to_string()
|
|
||||||
};
|
|
||||||
|
|
||||||
let out = Command::new("qemu-img")
|
|
||||||
.args(["create", "-f", "qcow2", "-F"])
|
|
||||||
.arg(&base_fmt)
|
|
||||||
.args(["-b"])
|
|
||||||
.arg(&base)
|
|
||||||
.arg(&overlay)
|
|
||||||
.arg(&size_arg)
|
|
||||||
.output()
|
|
||||||
.map_err(|e| miette::miette!("qemu-img not found or failed: {e}"))?;
|
|
||||||
if !out.status.success() {
|
|
||||||
return Err(miette::miette!(
|
|
||||||
"qemu-img create failed: {}",
|
|
||||||
String::from_utf8_lossy(&out.stderr)
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.into_diagnostic()??;
|
|
||||||
let _ = status; // appease compiler if unused
|
|
||||||
// Relax permissions on overlay so host qemu can access it
|
|
||||||
let _ = std::fs::set_permissions(&overlay, std::fs::Permissions::from_mode(0o666));
|
|
||||||
|
|
||||||
// Build NoCloud seed ISO if user_data provided, or synthesize from per-job SSH key
|
|
||||||
let mut seed_iso: Option<PathBuf> = None;
|
|
||||||
// Prefer spec-provided user-data; otherwise, if we have a per-job SSH public key, generate minimal cloud-config
|
|
||||||
let user_data_opt: Option<Vec<u8>> = if let Some(ref ud) = spec.user_data {
|
|
||||||
Some(ud.clone())
|
|
||||||
} else if let Some(ref pubkey) = ctx.ssh_public_key {
|
|
||||||
let s = format!(
|
|
||||||
r#"#cloud-config
|
|
||||||
users:
|
|
||||||
- default
|
|
||||||
ssh_authorized_keys:
|
|
||||||
- {ssh_pubkey}
|
|
||||||
"#,
|
|
||||||
ssh_pubkey = pubkey.trim()
|
|
||||||
);
|
|
||||||
Some(s.into_bytes())
|
|
||||||
} else {
|
|
||||||
None
|
|
||||||
};
|
|
||||||
|
|
||||||
if let Some(user_data) = user_data_opt {
|
|
||||||
let seed_dir = work_dir.join("seed");
|
|
||||||
tokio::fs::create_dir_all(&seed_dir)
|
|
||||||
.await
|
|
||||||
.into_diagnostic()?;
|
|
||||||
let ud_path = seed_dir.join("user-data");
|
|
||||||
let md_path = seed_dir.join("meta-data");
|
|
||||||
tokio::fs::write(&ud_path, &user_data)
|
|
||||||
.await
|
|
||||||
.into_diagnostic()?;
|
|
||||||
let meta = format!("instance-id: {}\nlocal-hostname: {}\n", id, id);
|
|
||||||
tokio::fs::write(&md_path, meta.as_bytes())
|
|
||||||
.await
|
|
||||||
.into_diagnostic()?;
|
|
||||||
|
|
||||||
// mkisofs or genisoimage
|
|
||||||
let iso_path = work_dir.join("seed.iso");
|
|
||||||
tokio::task::spawn_blocking({
|
|
||||||
let iso_path = iso_path.clone();
|
|
||||||
let seed_dir = seed_dir.clone();
|
|
||||||
move || -> miette::Result<()> {
|
|
||||||
let try_mk = |bin: &str| -> std::io::Result<std::process::Output> {
|
|
||||||
Command::new(bin)
|
|
||||||
.args(["-V", "cidata", "-J", "-R", "-o"])
|
|
||||||
.arg(&iso_path)
|
|
||||||
.arg(&seed_dir)
|
|
||||||
.output()
|
|
||||||
};
|
|
||||||
let out = try_mk("mkisofs")
|
|
||||||
.or_else(|_| try_mk("genisoimage"))
|
|
||||||
.map_err(|e| miette::miette!("mkisofs/genisoimage not found: {e}"))?;
|
|
||||||
if !out.status.success() {
|
|
||||||
return Err(miette::miette!(
|
|
||||||
"mkisofs failed: {}",
|
|
||||||
String::from_utf8_lossy(&out.stderr)
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.into_diagnostic()??;
|
|
||||||
seed_iso = Some(iso_path);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Relax permissions on seed ISO if created (readable by host qemu)
|
|
||||||
if let Some(ref p) = seed_iso {
|
|
||||||
let _ = std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o644));
|
|
||||||
}
|
|
||||||
|
|
||||||
// Serial console log file path (pre-create with permissive perms for libvirt)
|
|
||||||
let console_log = work_dir.join("console.log");
|
|
||||||
let _ = std::fs::OpenOptions::new()
|
|
||||||
.create(true)
|
|
||||||
.append(true)
|
|
||||||
.open(&console_log);
|
|
||||||
let _ = std::fs::set_permissions(&console_log, std::fs::Permissions::from_mode(0o666));
|
|
||||||
let console_log_str = console_log.display().to_string();
|
|
||||||
info!(domain = %id, console = %console_log_str, "serial console will be logged to file");
|
|
||||||
|
|
||||||
// Domain XML
|
|
||||||
let xml = {
|
|
||||||
let mem = spec.ram_mb;
|
|
||||||
let vcpus = spec.cpu;
|
|
||||||
let overlay_str = overlay.display().to_string();
|
|
||||||
let seed_str = seed_iso.as_ref().map(|p| p.display().to_string());
|
|
||||||
let net = self.network.clone();
|
|
||||||
let cdrom = seed_str.map(|p| format!("<disk type='file' device='cdrom'>\n <driver name='qemu' type='raw'/>\n <source file='{}'/>\n <target dev='hdb' bus='ide'/>\n <readonly/>\n</disk>", p)).unwrap_or_default();
|
|
||||||
format!(
|
|
||||||
"<domain type='kvm'>\n<name>{}</name>\n<memory unit='MiB'>{}</memory>\n<vcpu>{}</vcpu>\n<os>\n <type arch='x86_64' machine='pc'>hvm</type>\n <boot dev='hd'/>\n</os>\n<features><acpi/></features>\n<devices>\n <disk type='file' device='disk'>\n <driver name='qemu' type='qcow2' cache='none'/>\n <source file='{}'/>\n <target dev='vda' bus='virtio'/>\n </disk>\n {}\n <interface type='network'>\n <source network='{}'/>\n <model type='virtio'/>\n </interface>\n <graphics type='vnc' autoport='yes' listen='127.0.0.1'/>\n <serial type='pty'>\n <target port='0'/>\n </serial>\n <serial type='file'>\n <source path='{}'/>\n <target port='1'/>\n </serial>\n <console type='pty'>\n <target type='serial' port='0'/>\n </console>\n</devices>\n<on_poweroff>destroy</on_poweroff>\n<on_crash>destroy</on_crash>\n</domain>",
|
|
||||||
id, mem, vcpus, overlay_str, cdrom, net, console_log_str
|
|
||||||
)
|
|
||||||
};
|
|
||||||
|
|
||||||
// Define via virt crate
|
|
||||||
let uri2 = self.uri.clone();
|
|
||||||
let xml_clone = xml.clone();
|
|
||||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
|
||||||
use virt::{connect::Connect, domain::Domain};
|
|
||||||
let conn = Connect::open(Some(&uri2))
|
|
||||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
|
||||||
let _dom = Domain::define_xml(&conn, &xml_clone)
|
|
||||||
.map_err(|e| miette::miette!("define domain failed: {e}"))?;
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.into_diagnostic()??;
|
|
||||||
|
|
||||||
info!(domain = %id, image = ?spec.image_path, cpu = spec.cpu, ram_mb = spec.ram_mb, "libvirt prepared");
|
|
||||||
Ok(VmHandle {
|
|
||||||
id,
|
|
||||||
backend: BackendTag::Libvirt,
|
|
||||||
work_dir,
|
|
||||||
overlay_path: Some(overlay),
|
|
||||||
seed_iso_path: seed_iso,
|
|
||||||
console_socket: None,
|
|
||||||
ssh_host_port: None,
|
|
||||||
mac_addr: None,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn start(&self, vm: &VmHandle) -> Result<()> {
|
|
||||||
let id = vm.id.clone();
|
|
||||||
let uri = self.uri.clone();
|
|
||||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
|
||||||
use virt::{connect::Connect, domain::Domain};
|
|
||||||
let conn = Connect::open(Some(&uri))
|
|
||||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
|
||||||
// Lookup domain by name and start
|
|
||||||
let dom = Domain::lookup_by_name(&conn, &id)
|
|
||||||
.map_err(|e| miette::miette!("lookup domain failed: {e}"))?;
|
|
||||||
dom.create()
|
|
||||||
.map_err(|e| miette::miette!("domain start failed: {e}"))?;
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.into_diagnostic()??;
|
|
||||||
info!(domain = %vm.id, "libvirt started");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn stop(&self, vm: &VmHandle, t: Duration) -> Result<()> {
|
|
||||||
let id = vm.id.clone();
|
|
||||||
let uri = self.uri.clone();
|
|
||||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
|
||||||
use virt::{connect::Connect, domain::Domain};
|
|
||||||
let conn = Connect::open(Some(&uri))
|
|
||||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
|
||||||
let dom = Domain::lookup_by_name(&conn, &id)
|
|
||||||
.map_err(|e| miette::miette!("lookup domain failed: {e}"))?;
|
|
||||||
let _ = dom.shutdown();
|
|
||||||
let start = std::time::Instant::now();
|
|
||||||
while start.elapsed() < t {
|
|
||||||
match dom.is_active() {
|
|
||||||
Ok(false) => break,
|
|
||||||
_ => {}
|
|
||||||
}
|
|
||||||
std::thread::sleep(std::time::Duration::from_millis(500));
|
|
||||||
}
|
|
||||||
// Force destroy if still active
|
|
||||||
let _ = dom.destroy();
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.into_diagnostic()??;
|
|
||||||
info!(domain = %vm.id, "libvirt stopped");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
async fn suspend(&self, vm: &VmHandle) -> Result<()> {
|
|
||||||
let id = vm.id.clone();
|
|
||||||
let uri = self.uri.clone();
|
|
||||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
|
||||||
use virt::{connect::Connect, domain::Domain};
|
|
||||||
let conn = Connect::open(Some(&uri))
|
|
||||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
|
||||||
let dom = Domain::lookup_by_name(&conn, &id)
|
|
||||||
.map_err(|e| miette::miette!("lookup domain failed: {e}"))?;
|
|
||||||
dom.suspend()
|
|
||||||
.map_err(|e| miette::miette!("domain suspend failed: {e}"))?;
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.into_diagnostic()??;
|
|
||||||
info!(domain = %vm.id, "libvirt suspended");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn destroy(&self, vm: VmHandle) -> Result<()> {
|
|
||||||
let id = vm.id.clone();
|
|
||||||
let uri = self.uri.clone();
|
|
||||||
let id_for_task = id.clone();
|
|
||||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
|
||||||
use virt::{connect::Connect, domain::Domain};
|
|
||||||
let conn = Connect::open(Some(&uri))
|
|
||||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
|
||||||
if let Ok(dom) = Domain::lookup_by_name(&conn, &id_for_task) {
|
|
||||||
let _ = dom.undefine();
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.into_diagnostic()??;
|
|
||||||
// Cleanup files
|
|
||||||
if let Some(p) = vm.overlay_path.as_ref() {
|
|
||||||
let _ = tokio::fs::remove_file(p).await;
|
|
||||||
}
|
|
||||||
if let Some(p) = vm.seed_iso_path.as_ref() {
|
|
||||||
let _ = tokio::fs::remove_file(p).await;
|
|
||||||
}
|
|
||||||
let _ = tokio::fs::remove_dir_all(&vm.work_dir).await;
|
|
||||||
info!(domain = %id, "libvirt destroyed");
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
async fn state(&self, vm: &VmHandle) -> Result<VmState> {
|
|
||||||
let id = vm.id.clone();
|
|
||||||
let uri = self.uri.clone();
|
|
||||||
let active = tokio::task::spawn_blocking(move || -> miette::Result<bool> {
|
|
||||||
use virt::{connect::Connect, domain::Domain};
|
|
||||||
let conn = Connect::open(Some(&uri))
|
|
||||||
.map_err(|e| miette::miette!("libvirt connect failed: {e}"))?;
|
|
||||||
let dom = Domain::lookup_by_name(&conn, &id)
|
|
||||||
.map_err(|e| miette::miette!("lookup domain failed: {e}"))?;
|
|
||||||
let active = dom.is_active().unwrap_or(false);
|
|
||||||
Ok(active)
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.into_diagnostic()??;
|
|
||||||
Ok(if active {
|
|
||||||
VmState::Running
|
|
||||||
} else {
|
|
||||||
VmState::Stopped
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[cfg(target_os = "illumos")]
|
|
||||||
pub struct ZonesHypervisor;
|
|
||||||
|
|
||||||
#[cfg(target_os = "illumos")]
|
|
||||||
#[async_trait]
|
|
||||||
impl Hypervisor for ZonesHypervisor {
|
|
||||||
async fn prepare(&self, spec: &VmSpec, ctx: &JobContext) -> Result<VmHandle> {
|
|
||||||
use std::process::Command;
|
|
||||||
let id = format!("zone-{}", ctx.request_id);
|
|
||||||
// Create working directory under /var/lib/solstice-ci if possible
|
|
||||||
let work_dir = {
|
|
||||||
let base = std::path::Path::new("/var/lib/solstice-ci");
|
|
||||||
let dir = if base.exists() && base.is_dir() && std::fs::metadata(base).is_ok() {
|
|
||||||
base.join(&id)
|
|
||||||
} else {
|
|
||||||
std::env::temp_dir().join("solstice-zones").join(&id)
|
|
||||||
};
|
|
||||||
let _ = std::fs::create_dir_all(&dir);
|
|
||||||
#[cfg(unix)]
|
|
||||||
let _ = std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700));
|
|
||||||
dir
|
|
||||||
};
|
|
||||||
|
|
||||||
// Detect base image format
|
|
||||||
let base = spec.image_path.clone();
|
|
||||||
let base_fmt = tokio::task::spawn_blocking(move || -> miette::Result<String> {
|
|
||||||
let out = Command::new("qemu-img")
|
|
||||||
.args(["info", "--output=json"])
|
|
||||||
.arg(&base)
|
|
||||||
.output()
|
|
||||||
.map_err(|e| miette::miette!("qemu-img not found or failed: {e}"))?;
|
|
||||||
if !out.status.success() {
|
|
||||||
return Err(miette::miette!(
|
|
||||||
"qemu-img info failed: {}",
|
|
||||||
String::from_utf8_lossy(&out.stderr)
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let v: serde_json::Value = serde_json::from_slice(&out.stdout)
|
|
||||||
.map_err(|e| miette::miette!("parse qemu-img info json failed: {e}"))?;
|
|
||||||
Ok(v.get("format")
|
|
||||||
.and_then(|f| f.as_str())
|
|
||||||
.unwrap_or("raw")
|
|
||||||
.to_string())
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.into_diagnostic()??;
|
|
||||||
|
|
||||||
// Ensure raw image for bhyve: convert if needed
|
|
||||||
let raw_path = if base_fmt != "raw" {
|
|
||||||
let out_path = work_dir.join("disk.raw");
|
|
||||||
let src = spec.image_path.clone();
|
|
||||||
let dst = out_path.clone();
|
|
||||||
tokio::task::spawn_blocking(move || -> miette::Result<()> {
|
|
||||||
let out = Command::new("qemu-img")
|
|
||||||
.args(["convert", "-O", "raw"])
|
|
||||||
.arg(&src)
|
|
||||||
.arg(&dst)
|
|
||||||
.output()
|
|
||||||
.map_err(|e| miette::miette!("qemu-img convert failed to start: {e}"))?;
|
|
||||||
if !out.status.success() {
|
|
||||||
return Err(miette::miette!(
|
|
||||||
"qemu-img convert failed: {}",
|
|
||||||
String::from_utf8_lossy(&out.stderr)
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Ok(())
|
|
||||||
})
|
|
||||||
.await
|
|
||||||
.into_diagnostic()??;
|
|
||||||
info!(label = %spec.label, src = ?spec.image_path, out = ?out_path, "converted image to raw for bhyve");
|
|
||||||
out_path
|
|
||||||
} else {
|
|
||||||
spec.image_path.clone()
|
|
||||||
};
|
|
||||||
|
|
||||||
// Seed ISO creation left to future; for now, return handle with path in overlay_path
|
|
||||||
Ok(VmHandle {
|
|
||||||
id,
|
|
||||||
backend: BackendTag::Zones,
|
|
||||||
work_dir,
|
|
||||||
overlay_path: Some(raw_path),
|
|
||||||
seed_iso_path: None,
|
|
||||||
console_socket: None,
|
|
||||||
ssh_host_port: None,
|
|
||||||
mac_addr: None,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
async fn start(&self, _vm: &VmHandle) -> Result<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
async fn stop(&self, _vm: &VmHandle, _t: Duration) -> Result<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
async fn destroy(&self, _vm: VmHandle) -> Result<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
async fn suspend(&self, _vm: &VmHandle) -> Result<()> {
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -17,7 +17,7 @@ use tracing::{debug, info, warn};
|
||||||
use crate::error::OrchestratorError;
|
use crate::error::OrchestratorError;
|
||||||
use crate::persist::{JobState, Persist};
|
use crate::persist::{JobState, Persist};
|
||||||
use config::OrchestratorConfig;
|
use config::OrchestratorConfig;
|
||||||
use hypervisor::{JobContext, RouterHypervisor, VmSpec};
|
use hypervisor::{JobContext, VmSpec};
|
||||||
use scheduler::{ExecConfig, SchedItem, Scheduler};
|
use scheduler::{ExecConfig, SchedItem, Scheduler};
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use tokio::sync::Notify;
|
use tokio::sync::Notify;
|
||||||
|
|
@ -73,13 +73,10 @@ struct Opts {
|
||||||
#[arg(long, env = "AMQP_PREFETCH")]
|
#[arg(long, env = "AMQP_PREFETCH")]
|
||||||
amqp_prefetch: Option<u16>,
|
amqp_prefetch: Option<u16>,
|
||||||
|
|
||||||
/// Libvirt URI (Linux only)
|
/// Network bridge name for TAP networking (e.g., "virbr0"). If empty, uses
|
||||||
#[arg(long, env = "LIBVIRT_URI", default_value = "qemu:///system")]
|
/// user-mode (SLIRP) networking with SSH port forwarding — recommended for containers.
|
||||||
libvirt_uri: String,
|
#[arg(long, env = "NETWORK_BRIDGE")]
|
||||||
|
network_bridge: Option<String>,
|
||||||
/// Libvirt network name (Linux only)
|
|
||||||
#[arg(long, env = "LIBVIRT_NETWORK", default_value = "default")]
|
|
||||||
libvirt_network: String,
|
|
||||||
|
|
||||||
/// OTLP endpoint (e.g., http://localhost:4317)
|
/// OTLP endpoint (e.g., http://localhost:4317)
|
||||||
#[arg(long, env = "OTEL_EXPORTER_OTLP_ENDPOINT")]
|
#[arg(long, env = "OTEL_EXPORTER_OTLP_ENDPOINT")]
|
||||||
|
|
@ -188,8 +185,8 @@ async fn main() -> Result<()> {
|
||||||
// Capacity settings
|
// Capacity settings
|
||||||
let capacity_map = parse_capacity_map(opts.capacity_map.as_deref());
|
let capacity_map = parse_capacity_map(opts.capacity_map.as_deref());
|
||||||
|
|
||||||
// Build hypervisor router
|
// Build hypervisor via vm-manager adapter (QEMU direct, no libvirt)
|
||||||
let router = RouterHypervisor::build(opts.libvirt_uri.clone(), opts.libvirt_network.clone());
|
let router = vm_adapter::VmManagerAdapter::build(opts.network_bridge.clone());
|
||||||
|
|
||||||
// Initialize persistence (optional). Skip when requested for faster startup.
|
// Initialize persistence (optional). Skip when requested for faster startup.
|
||||||
let persist = if opts.skip_persistence {
|
let persist = if opts.skip_persistence {
|
||||||
|
|
@ -223,8 +220,6 @@ async fn main() -> Result<()> {
|
||||||
runner_illumos_path: opts.runner_illumos_path.clone(),
|
runner_illumos_path: opts.runner_illumos_path.clone(),
|
||||||
ssh_connect_timeout_secs: opts.ssh_connect_timeout_secs,
|
ssh_connect_timeout_secs: opts.ssh_connect_timeout_secs,
|
||||||
boot_wait_secs: opts.boot_wait_secs,
|
boot_wait_secs: opts.boot_wait_secs,
|
||||||
libvirt_uri: opts.libvirt_uri.clone(),
|
|
||||||
libvirt_network: opts.libvirt_network.clone(),
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let sched = Scheduler::new(
|
let sched = Scheduler::new(
|
||||||
|
|
@ -357,7 +352,7 @@ async fn main() -> Result<()> {
|
||||||
cpu,
|
cpu,
|
||||||
ram_mb,
|
ram_mb,
|
||||||
disk_gb,
|
disk_gb,
|
||||||
network: None, // libvirt network handled in backend
|
network: None, // vm-manager handles networking
|
||||||
nocloud: image.nocloud,
|
nocloud: image.nocloud,
|
||||||
user_data: Some(make_cloud_init_userdata(
|
user_data: Some(make_cloud_init_userdata(
|
||||||
&job.repo_url,
|
&job.repo_url,
|
||||||
|
|
|
||||||
|
|
@ -40,8 +40,6 @@ pub struct ExecConfig {
|
||||||
pub runner_illumos_path: String,
|
pub runner_illumos_path: String,
|
||||||
pub ssh_connect_timeout_secs: u64,
|
pub ssh_connect_timeout_secs: u64,
|
||||||
pub boot_wait_secs: u64,
|
pub boot_wait_secs: u64,
|
||||||
pub libvirt_uri: String,
|
|
||||||
pub libvirt_network: String,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Strip ANSI escape sequences (colors/control) from a string for clean log storage.
|
// Strip ANSI escape sequences (colors/control) from a string for clean log storage.
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue