diff --git a/Cargo.lock b/Cargo.lock index 6379d23..ec82d4f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -133,6 +133,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -217,6 +223,7 @@ dependencies = [ "iana-time-zone", "js-sys", "num-traits", + "serde", "wasm-bindgen", "windows-link", ] @@ -283,6 +290,26 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const_format" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7faa7469a93a566e9ccc1c73fe783b4a65c274c5ace346038dca9c39fe0030ad" +dependencies = [ + "const_format_proc_macros", +] + +[[package]] +name = "const_format_proc_macros" +version = "0.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -365,6 +392,41 @@ dependencies = [ "syn", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn", +] + [[package]] name = "data-encoding" version = "2.10.0" @@ -381,6 +443,37 @@ dependencies = [ "zeroize", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "digest" version = "0.10.7" @@ -535,12 +628,33 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -662,6 +776,18 @@ dependencies = [ "wasip3", ] +[[package]] +name = "getset" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912" +dependencies = [ + "proc-macro-error2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "gimli" version = "0.32.3" @@ -719,6 +845,15 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-auth" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822" +dependencies = [ + "memchr", +] + [[package]] name = "http-body" version = "1.0.1" @@ -786,13 +921,29 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-util", @@ -920,6 +1071,12 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "1.1.0" @@ -1029,6 +1186,21 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "jwt" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6204285f77fe7d9784db3fdc449ecce1a0114927a51d5a41c4c7a292011c015f" +dependencies = [ + "base64 0.13.1", + "crypto-common", + "digest", + "hmac", + "serde", + "serde_json", + "sha2", +] + [[package]] name = "kdl" version = "6.5.0" @@ -1201,6 +1373,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "native-tls" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5d26952a508f321b4d3d2e80e78fc2603eaefcdf0c30783867f19586518bdc" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1309,6 +1498,60 @@ dependencies = [ "memchr", ] +[[package]] +name = "oci-client" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b74df13319e08bc386d333d3dc289c774c88cc543cae31f5347db07b5ec2172" +dependencies = [ + "bytes", + "chrono", + "futures-util", + "http", + "http-auth", + "jwt", + "lazy_static", + "oci-spec", + "olpc-cjson", + "regex", + "reqwest", + "serde", + "serde_json", + "sha2", + "thiserror", + "tokio", + "tracing", + "unicase", +] + +[[package]] +name = "oci-spec" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc3da52b83ce3258fbf29f66ac784b279453c2ac3c22c5805371b921ede0d308" +dependencies = [ + "const_format", + "derive_builder", + "getset", + "regex", + "serde", + "serde_json", + "strum", + "strum_macros", + "thiserror", +] + +[[package]] +name = "olpc-cjson" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083" +dependencies = [ + "serde", + "serde_json", + "unicode-normalization", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -1321,6 +1564,32 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags 2.11.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.2.1" @@ -1503,6 +1772,28 @@ dependencies = [ "elliptic-curve", ] +[[package]] +name = "proc-macro-error-attr2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "proc-macro-error2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802" +dependencies = [ + "proc-macro-error-attr2", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1695,7 +1986,7 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-core", "futures-util", @@ -1704,9 +1995,11 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", + "hyper-tls", "hyper-util", "js-sys", "log", + "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -1718,6 +2011,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-native-tls", "tokio-rustls", "tokio-util", "tower", @@ -2142,6 +2436,24 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" + +[[package]] +name = "strum_macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -2314,6 +2626,16 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -2400,6 +2722,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2484,6 +2807,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" +[[package]] +name = "unicase" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" + [[package]] name = "unicode-ident" version = "1.0.23" @@ -2496,6 +2825,15 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b09c83c3c29d37506a3e260c08c03743a6bb66a9cd432c6934ab501a190571f" +[[package]] +name = "unicode-normalization" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-width" version = "0.1.14" @@ -2590,6 +2928,7 @@ dependencies = [ "kdl", "libc", "miette", + "oci-client", "reqwest", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index 54aeb00..75147a4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,3 +37,4 @@ zstd = "0.13" dirs = "6" kdl = "6" ssh-key = { version = "0.6", features = ["ed25519", "rand_core", "getrandom"] } +oci-client = "0.15" diff --git a/crates/vm-manager/Cargo.toml b/crates/vm-manager/Cargo.toml index 78a7ffb..15d5630 100644 --- a/crates/vm-manager/Cargo.toml +++ b/crates/vm-manager/Cargo.toml @@ -27,6 +27,9 @@ kdl.workspace = true # Optional pure-Rust ISO generation isobemak = { version = "0.2", optional = true } +# OCI +oci-client.workspace = true + # SSH ssh2 = "0.9" ssh-key.workspace = true diff --git a/crates/vm-manager/src/error.rs b/crates/vm-manager/src/error.rs index 528d24d..696bf13 100644 --- a/crates/vm-manager/src/error.rs +++ b/crates/vm-manager/src/error.rs @@ -157,6 +157,13 @@ pub enum VmError { detail: String, }, + #[error("failed to pull OCI artifact {reference}: {detail}")] + #[diagnostic( + code(vm_manager::oci::pull_failed), + help("check that the OCI reference is correct and the registry is reachable. For ghcr.io, ensure GITHUB_TOKEN is set in the environment.") + )] + OciPullFailed { reference: String, detail: String }, + #[error(transparent)] #[diagnostic(code(vm_manager::io))] Io(#[from] std::io::Error), diff --git a/crates/vm-manager/src/image.rs b/crates/vm-manager/src/image.rs index 05679d1..faa5711 100644 --- a/crates/vm-manager/src/image.rs +++ b/crates/vm-manager/src/image.rs @@ -64,6 +64,30 @@ impl ImageManager { } } + /// Pull a QCOW2 image from an OCI registry into the cache directory. + pub async fn pull_oci(&self, reference: &str, name: Option<&str>) -> Result { + let file_name = name + .map(|n| format!("{n}.qcow2")) + .unwrap_or_else(|| { + let sanitized = reference.replace('/', "_").replace(':', "_"); + format!("{sanitized}.qcow2") + }); + let dest = self.cache.join(&file_name); + if dest.exists() { + info!(reference, dest = %dest.display(), "OCI image already cached; skipping pull"); + return Ok(dest); + } + + if let Some(parent) = dest.parent() { + tokio::fs::create_dir_all(parent).await?; + } + + let data = crate::oci::pull_qcow2(reference).await?; + tokio::fs::write(&dest, &data).await?; + info!(reference, dest = %dest.display(), "OCI artifact cached"); + Ok(dest) + } + /// Pull an image from a URL into the cache directory, returning the cached path. pub async fn pull(&self, url: &str, name: Option<&str>) -> Result { let file_name = name.map(|n| n.to_string()).unwrap_or_else(|| { diff --git a/crates/vm-manager/src/lib.rs b/crates/vm-manager/src/lib.rs index 6f1eb09..c04bfd0 100644 --- a/crates/vm-manager/src/lib.rs +++ b/crates/vm-manager/src/lib.rs @@ -2,6 +2,7 @@ pub mod backends; pub mod cloudinit; pub mod error; pub mod image; +pub mod oci; pub mod provision; pub mod ssh; pub mod traits; diff --git a/crates/vm-manager/src/oci.rs b/crates/vm-manager/src/oci.rs new file mode 100644 index 0000000..534e72c --- /dev/null +++ b/crates/vm-manager/src/oci.rs @@ -0,0 +1,93 @@ +use oci_client::client::{ClientConfig, ClientProtocol}; +use oci_client::secrets::RegistryAuth; +use oci_client::{Client, Reference}; +use tracing::info; + +use crate::error::{Result, VmError}; + +const QCOW2_LAYER_MEDIA_TYPE: &str = "application/vnd.cloudnebula.qcow2.layer.v1"; + +/// Pull a QCOW2 image stored as an OCI artifact from a registry. +pub async fn pull_qcow2(reference_str: &str) -> Result> { + let reference: Reference = reference_str.parse().map_err(|e: oci_client::ParseError| { + VmError::OciPullFailed { + reference: reference_str.to_string(), + detail: format!("invalid OCI reference: {e}"), + } + })?; + + let auth = resolve_auth(&reference); + + let client_config = ClientConfig { + protocol: ClientProtocol::Https, + ..Default::default() + }; + let client = Client::new(client_config); + + info!(reference = %reference, "Pulling QCOW2 artifact from OCI registry"); + + let image_data = client + .pull( + &reference, + &auth, + vec![QCOW2_LAYER_MEDIA_TYPE, "application/octet-stream"], + ) + .await + .map_err(|e| VmError::OciPullFailed { + reference: reference_str.to_string(), + detail: e.to_string(), + })?; + + // Find the QCOW2 layer + let layer = image_data + .layers + .into_iter() + .next() + .ok_or_else(|| VmError::OciPullFailed { + reference: reference_str.to_string(), + detail: "artifact contains no layers".to_string(), + })?; + + info!( + reference = %reference, + size_bytes = layer.data.len(), + "QCOW2 artifact pulled successfully" + ); + + Ok(layer.data) +} + +/// Resolve authentication for the given registry. +/// Uses GITHUB_TOKEN for ghcr.io, Anonymous for everything else. +fn resolve_auth(reference: &Reference) -> RegistryAuth { + let registry = reference.registry(); + if registry == "ghcr.io" { + if let Ok(token) = std::env::var("GITHUB_TOKEN") { + return RegistryAuth::Basic("_token".to_string(), token); + } + } + RegistryAuth::Anonymous +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_resolve_auth_ghcr_without_token() { + // Without GITHUB_TOKEN set, ghcr.io should use Anonymous + // SAFETY: This test is not run in parallel with other tests that + // depend on GITHUB_TOKEN. + unsafe { std::env::remove_var("GITHUB_TOKEN") }; + let reference: Reference = "ghcr.io/test/image:latest".parse().unwrap(); + let auth = resolve_auth(&reference); + assert!(matches!(auth, RegistryAuth::Anonymous)); + } + + #[test] + fn test_resolve_auth_other_registry() { + let reference: Reference = "docker.io/library/ubuntu:latest".parse().unwrap(); + let auth = resolve_auth(&reference); + assert!(matches!(auth, RegistryAuth::Anonymous)); + } +} diff --git a/crates/vm-manager/src/vmfile.rs b/crates/vm-manager/src/vmfile.rs index efca51d..9338d12 100644 --- a/crates/vm-manager/src/vmfile.rs +++ b/crates/vm-manager/src/vmfile.rs @@ -41,6 +41,7 @@ pub struct VmDef { pub enum ImageSource { Local(String), Url(String), + Oci(String), } /// Network mode as declared in the VMFile. @@ -229,6 +230,7 @@ fn parse_vm_def(name: &str, doc: &KdlDocument) -> Result { let image = match (local_image, url_image) { (Some(path), None) => ImageSource::Local(path), + (None, Some(url)) if url.starts_with("oci://") => ImageSource::Oci(url[6..].to_string()), (None, Some(url)) => ImageSource::Url(url), (Some(_), Some(_)) => { return Err(VmError::VmFileValidation { @@ -451,6 +453,10 @@ pub async fn resolve(def: &VmDef, base_dir: &Path) -> Result { let mgr = ImageManager::new(); mgr.pull(url, Some(&def.name)).await? } + ImageSource::Oci(oci_ref) => { + let mgr = ImageManager::new(); + mgr.pull_oci(oci_ref, Some(&def.name)).await? + } }; // Network @@ -780,6 +786,22 @@ vm "dup" { assert!(msg.contains("duplicate"), "got: {msg}"); } + #[test] + fn parse_oci_image_source() { + let kdl = r#" +vm "ci" { + image-url "oci://ghcr.io/cloudnebulaproject/ubuntu-rust:latest" +} +"#; + let tmp = tempfile::NamedTempFile::with_suffix(".kdl").unwrap(); + std::fs::write(tmp.path(), kdl).unwrap(); + + let vmfile = parse(tmp.path()).unwrap(); + assert!( + matches!(vmfile.vms[0].image, ImageSource::Oci(ref r) if r == "ghcr.io/cloudnebulaproject/ubuntu-rust:latest") + ); + } + #[test] fn expand_tilde_works() { let home = dirs::home_dir().unwrap_or_else(|| PathBuf::from("/root"));