From 19c8379fc6c371b8566fdd21b96bfb57b83e1993 Mon Sep 17 00:00:00 2001 From: Till Wegmueller Date: Sun, 15 Feb 2026 17:17:30 +0100 Subject: [PATCH] Add builder VM support for cross-platform and unprivileged builds Introduce the forge-builder crate that automatically delegates builds to an ephemeral VM when the host can't build locally (e.g., QCOW2 targets without root, or OmniOS images on Linux). The builder detects these conditions, spins up a VM via vm-manager with user-mode networking, uploads inputs, streams the remote build output, and retrieves artifacts. Key changes: - New forge-builder crate with detection, binary resolution, VM lifecycle management, file transfer, and miette diagnostic errors - BuilderNode added to spec-parser schema for per-spec VM config - --local and --use-builder CLI flags on the build command - Feature-gated (default on) integration in forger CLI - Fix ext4 QCOW2 grub-install failure by using absolute paths in chroot - Improve debootstrap to pass --components and write full sources.list Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 1019 +++++++++++++++++- Cargo.toml | 10 + crates/forge-builder/Cargo.toml | 24 + crates/forge-builder/src/binary.rs | 192 ++++ crates/forge-builder/src/config.rs | 89 ++ crates/forge-builder/src/detect.rs | 92 ++ crates/forge-builder/src/error.rs | 71 ++ crates/forge-builder/src/lib.rs | 96 ++ crates/forge-builder/src/lifecycle.rs | 194 ++++ crates/forge-builder/src/transfer.rs | 149 +++ crates/forge-engine/src/phase1/mod.rs | 20 +- crates/forge-engine/src/phase2/qcow2_ext4.rs | 4 +- crates/forge-engine/src/tools/apt.rs | 52 +- crates/forger/Cargo.toml | 5 + crates/forger/src/commands/build.rs | 31 + crates/forger/src/main.rs | 12 +- crates/spec-parser/src/lib.rs | 63 ++ crates/spec-parser/src/resolve.rs | 5 + crates/spec-parser/src/schema.rs | 16 + 19 files changed, 2125 insertions(+), 19 deletions(-) create mode 100644 crates/forge-builder/Cargo.toml create mode 100644 crates/forge-builder/src/binary.rs create mode 100644 crates/forge-builder/src/config.rs create mode 100644 crates/forge-builder/src/detect.rs create mode 100644 crates/forge-builder/src/error.rs create mode 100644 crates/forge-builder/src/lib.rs create mode 100644 crates/forge-builder/src/lifecycle.rs create mode 100644 crates/forge-builder/src/transfer.rs diff --git a/Cargo.lock b/Cargo.lock index 4bc584b..0f8a700 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -145,6 +145,12 @@ dependencies = [ "backtrace", ] +[[package]] +name = "base16ct" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" + [[package]] name = "base64" version = "0.13.1" @@ -163,6 +169,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + [[package]] name = "bitflags" version = "2.11.0" @@ -203,6 +215,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -212,6 +226,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.43" @@ -235,6 +255,16 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.58" @@ -294,6 +324,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "const_format" version = "0.2.35" @@ -348,6 +384,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-bigint" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +dependencies = [ + "generic-array", + "rand_core 0.6.4", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -358,6 +406,32 @@ dependencies = [ "typenum", ] +[[package]] +name = "curve25519-dalek" +version = "4.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "digest", + "fiat-crypto", + "rustc_version", + "subtle", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.115", +] + [[package]] name = "darling" version = "0.20.11" @@ -393,6 +467,22 @@ dependencies = [ "syn 2.0.115", ] +[[package]] +name = "data-encoding" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "zeroize", +] + [[package]] name = "derive_builder" version = "0.20.2" @@ -431,10 +521,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", + "const-oid", "crypto-common", "subtle", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -446,6 +558,60 @@ dependencies = [ "syn 2.0.115", ] +[[package]] +name = "ecdsa" +version = "0.16.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +dependencies = [ + "der", + "digest", + "elliptic-curve", + "rfc6979", + "signature", + "spki", +] + +[[package]] +name = "ed25519" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" +dependencies = [ + "signature", +] + +[[package]] +name = "ed25519-dalek" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70e796c081cee67dc755e1a36a0a172b897fab85fc3f6bc48307991f64e4eca9" +dependencies = [ + "curve25519-dalek", + "ed25519", + "sha2", + "subtle", +] + +[[package]] +name = "elliptic-curve" +version = "0.13.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +dependencies = [ + "base16ct", + "crypto-bigint", + "digest", + "ff", + "generic-array", + "group", + "pkcs8", + "rand_core 0.6.4", + "sec1", + "subtle", + "zeroize", +] + [[package]] name = "encode_unicode" version = "1.0.0" @@ -474,6 +640,22 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "ff" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +dependencies = [ + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "fiat-crypto" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28dea519a9695b9977216879a3ebfddf92f1c08c05d984f8996aecd6ecdc811d" + [[package]] name = "filetime" version = "0.2.27" @@ -528,6 +710,26 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "forge-builder" +version = "0.1.0" +dependencies = [ + "dirs", + "libc", + "miette 7.6.0", + "reqwest", + "serde_json", + "spec-parser", + "ssh-key", + "ssh2", + "tar", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tracing", + "vm-manager", +] + [[package]] name = "forge-engine" version = "0.1.0" @@ -575,6 +777,7 @@ name = "forger" version = "0.1.0" dependencies = [ "clap", + "forge-builder", "forge-engine", "forge-oci", "indicatif", @@ -665,6 +868,34 @@ checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" dependencies = [ "typenum", "version_check", + "zeroize", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", ] [[package]] @@ -698,6 +929,17 @@ version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" +[[package]] +name = "group" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +dependencies = [ + "ff", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -822,6 +1064,23 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.6.0" @@ -1024,6 +1283,15 @@ dependencies = [ "web-time", ] +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -1058,6 +1326,16 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.85" @@ -1083,6 +1361,17 @@ dependencies = [ "sha2", ] +[[package]] +name = "kdl" +version = "6.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81a29e7b50079ff44549f68c0becb1c73d7f6de2a4ea952da77966daf3d4761e" +dependencies = [ + "miette 7.6.0", + "num", + "winnow", +] + [[package]] name = "knuffel" version = "3.2.0" @@ -1115,6 +1404,9 @@ name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +dependencies = [ + "spin", +] [[package]] name = "leb128fmt" @@ -1128,6 +1420,12 @@ version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.12" @@ -1136,7 +1434,33 @@ checksum = "3d0b95e02c851351f877147b7deea7b1afb1df71b63aa5f8270716e0c5720616" dependencies = [ "bitflags", "libc", - "redox_syscall", + "redox_syscall 0.7.1", +] + +[[package]] +name = "libssh2-sys" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "220e4f05ad4a218192533b300327f5150e809b54c4ec83b5a1d91833601811b9" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", ] [[package]] @@ -1151,12 +1475,27 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "matchers" version = "0.2.0" @@ -1272,6 +1611,86 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-bigint-dig" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e661dda6640fad38e827a6d4a310ff4763082116fe217f279885c97f511bb0b7" +dependencies = [ + "lazy_static", + "libm", + "num-integer", + "num-iter", + "num-traits", + "rand 0.8.5", + "smallvec", + "zeroize", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1279,6 +1698,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1406,12 +1826,88 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "owo-colors" version = "4.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c6901729fa79e91a0913333229e9ca5dc725089d1c363b2f4b4760709dc4a52" +[[package]] +name = "p256" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "p384" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" +dependencies = [ + "ecdsa", + "elliptic-curve", + "primeorder", + "sha2", +] + +[[package]] +name = "p521" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" +dependencies = [ + "base16ct", + "ecdsa", + "elliptic-curve", + "primeorder", + "rand_core 0.6.4", + "sha2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1430,6 +1926,27 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkcs1" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" +dependencies = [ + "der", + "pkcs8", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -1451,6 +1968,15 @@ dependencies = [ "zerovec", ] +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "prettyplease" version = "0.2.37" @@ -1461,6 +1987,15 @@ dependencies = [ "syn 2.0.115", ] +[[package]] +name = "primeorder" +version = "0.13.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +dependencies = [ + "elliptic-curve", +] + [[package]] name = "proc-macro-error" version = "1.0.4" @@ -1516,6 +2051,61 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror 2.0.18", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.4", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.18", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.44" @@ -1531,6 +2121,73 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.7.1" @@ -1540,6 +2197,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 2.0.18", +] + [[package]] name = "regex" version = "1.12.3" @@ -1583,6 +2251,7 @@ dependencies = [ "http-body", "http-body-util", "hyper", + "hyper-rustls", "hyper-tls", "hyper-util", "js-sys", @@ -1590,6 +2259,9 @@ dependencies = [ "native-tls", "percent-encoding", "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", "rustls-pki-types", "serde", "serde_json", @@ -1597,6 +2269,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", + "tokio-rustls", "tokio-util", "tower", "tower-http", @@ -1608,12 +2281,72 @@ dependencies = [ "web-sys", ] +[[package]] +name = "rfc6979" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +dependencies = [ + "hmac", + "subtle", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rsa" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" +dependencies = [ + "const-oid", + "digest", + "num-bigint-dig", + "num-integer", + "num-traits", + "pkcs1", + "pkcs8", + "rand_core 0.6.4", + "sha2", + "signature", + "spki", + "subtle", + "zeroize", +] + [[package]] name = "rustc-demangle" version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rustix" version = "1.1.3" @@ -1627,15 +2360,53 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" +dependencies = [ + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + [[package]] name = "rustls-pki-types" version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" dependencies = [ + "web-time", "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -1666,6 +2437,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "sec1" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +dependencies = [ + "base16ct", + "der", + "generic-array", + "pkcs8", + "subtle", + "zeroize", +] + [[package]] name = "security-framework" version = "3.6.0" @@ -1750,6 +2541,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1786,6 +2588,16 @@ dependencies = [ "libc", ] +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "digest", + "rand_core 0.6.4", +] + [[package]] name = "simd-adler32" version = "0.3.8" @@ -1825,6 +2637,76 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "ssh-cipher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caac132742f0d33c3af65bfcde7f6aa8f62f0e991d80db99149eb9d44708784f" +dependencies = [ + "cipher", + "ssh-encoding", +] + +[[package]] +name = "ssh-encoding" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb9242b9ef4108a78e8cd1a2c98e193ef372437f8c22be363075233321dd4a15" +dependencies = [ + "base64ct", + "pem-rfc7468", + "sha2", +] + +[[package]] +name = "ssh-key" +version = "0.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b86f5297f0f04d08cabaa0f6bff7cb6aec4d9c3b49d87990d63da9d9156a8c3" +dependencies = [ + "ed25519-dalek", + "p256", + "p384", + "p521", + "rand_core 0.6.4", + "rsa", + "sec1", + "sha2", + "signature", + "ssh-cipher", + "ssh-encoding", + "subtle", + "zeroize", +] + +[[package]] +name = "ssh2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f84d13b3b8a0d4e91a2629911e951db1bb8671512f5c09d7d4ba34500ba68c8" +dependencies = [ + "bitflags", + "libc", + "libssh2-sys", + "parking_lot", +] + [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1942,7 +2824,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0136791f7c95b1f6dd99f9cc786b91bb81c3800b639b3478e561ddb7be95e5f1" dependencies = [ "fastrand", - "getrandom", + "getrandom 0.4.1", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2079,6 +2961,28 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" +dependencies = [ + "futures-util", + "log", + "tokio", + "tungstenite", +] + [[package]] name = "tokio-util" version = "0.7.18" @@ -2205,6 +3109,23 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "tungstenite" +version = "0.26.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" +dependencies = [ + "bytes", + "data-encoding", + "http", + "httparse", + "log", + "rand 0.9.2", + "sha1", + "thiserror 2.0.18", + "utf-8", +] + [[package]] name = "typenum" version = "1.19.0" @@ -2262,6 +3183,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -2274,6 +3201,12 @@ dependencies = [ "serde", ] +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -2286,6 +3219,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" +dependencies = [ + "getrandom 0.4.1", + "js-sys", + "serde_core", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -2304,6 +3249,30 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "vm-manager" +version = "0.1.0" +dependencies = [ + "dirs", + "futures-util", + "kdl", + "libc", + "miette 7.6.0", + "oci-client", + "reqwest", + "serde", + "serde_json", + "ssh-key", + "ssh2", + "tempfile", + "thiserror 2.0.18", + "tokio", + "tokio-tungstenite", + "tracing", + "uuid", + "zstd", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -2541,6 +3510,15 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.6", +] + [[package]] name = "windows-sys" version = "0.59.0" @@ -2697,6 +3675,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.6.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.46.0" @@ -2915,3 +3902,31 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index 298cd3d..f8d62be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,7 @@ members = [ "crates/spec-parser", "crates/forge-oci", "crates/forge-engine", + "crates/forge-builder", "crates/forger", ] @@ -52,7 +53,16 @@ tempfile = "3" bytesize = "2" indicatif = "0.17" +# Builder VM support +vm-manager = { path = "../vm-manager/crates/vm-manager" } +reqwest = { version = "0.12", default-features = false, features = ["rustls-tls-native-roots", "stream"] } +libc = "0.2" +dirs = "6" +ssh2 = "0.9" +ssh-key = { version = "0.6", features = ["ed25519", "rand_core", "getrandom"] } + # Internal crates spec-parser = { path = "crates/spec-parser" } forge-oci = { path = "crates/forge-oci" } forge-engine = { path = "crates/forge-engine" } +forge-builder = { path = "crates/forge-builder" } diff --git a/crates/forge-builder/Cargo.toml b/crates/forge-builder/Cargo.toml new file mode 100644 index 0000000..9bc6b47 --- /dev/null +++ b/crates/forge-builder/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "forge-builder" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true + +[dependencies] +spec-parser = { workspace = true } +vm-manager = { workspace = true } +miette = { workspace = true } +thiserror = { workspace = true } +tokio = { workspace = true } +tracing = { workspace = true } +reqwest = { workspace = true } +ssh2 = { workspace = true } +ssh-key = { workspace = true } +libc = { workspace = true } +dirs = { workspace = true } +tempfile = { workspace = true } +serde_json = { workspace = true } +tar = { workspace = true } + +[dev-dependencies] +tempfile = { workspace = true } diff --git a/crates/forge-builder/src/binary.rs b/crates/forge-builder/src/binary.rs new file mode 100644 index 0000000..d9ea08c --- /dev/null +++ b/crates/forge-builder/src/binary.rs @@ -0,0 +1,192 @@ +use std::path::PathBuf; + +use spec_parser::schema::DistroFamily; +use tracing::info; + +use crate::error::BuilderError; + +/// Resolved forger binary for use inside a builder VM. +pub struct ResolvedBinary { + pub path: PathBuf, +} + +/// Map a distro family to the Rust target triple needed inside the builder VM. +pub fn target_triple(distro: &DistroFamily) -> &'static str { + match distro { + DistroFamily::OmniOS => "x86_64-unknown-illumos", + DistroFamily::Ubuntu => "x86_64-unknown-linux-gnu", + } +} + +/// Detect whether the current executable is a dev build (running from cargo target dir). +pub fn is_dev_build() -> bool { + std::env::current_exe() + .ok() + .and_then(|p| p.to_str().map(|s| s.contains("/target/"))) + .unwrap_or(false) +} + +/// Find the workspace root by walking up from the current exe looking for a workspace Cargo.toml. +fn find_workspace_root() -> Option { + let exe = std::env::current_exe().ok()?; + let mut dir = exe.parent()?; + + loop { + let cargo_toml = dir.join("Cargo.toml"); + if cargo_toml.exists() { + // Check if it's a workspace root (contains [workspace]) + if let Ok(content) = std::fs::read_to_string(&cargo_toml) { + if content.contains("[workspace]") { + return Some(dir.to_path_buf()); + } + } + } + dir = dir.parent()?; + } +} + +/// Resolve the forger binary path for the given distro. +/// +/// In dev mode: looks for cross-compiled binary in the workspace target directory. +/// In release mode: downloads from GitHub releases (cached locally). +pub async fn resolve_forger_binary(distro: &DistroFamily) -> Result { + let triple = target_triple(distro); + + if is_dev_build() { + resolve_dev_binary(triple) + } else { + resolve_release_binary(triple).await + } +} + +fn resolve_dev_binary(triple: &str) -> Result { + let workspace_root = find_workspace_root().ok_or_else(|| BuilderError::BinaryNotFound { + target_triple: triple.to_string(), + path: "".to_string(), + })?; + + let binary_path = workspace_root + .join("target") + .join(triple) + .join("release") + .join("forger"); + + if !binary_path.exists() { + return Err(BuilderError::BinaryNotFound { + target_triple: triple.to_string(), + path: binary_path.display().to_string(), + }); + } + + info!(path = %binary_path.display(), triple, "Using dev cross-compiled forger binary"); + Ok(ResolvedBinary { path: binary_path }) +} + +async fn resolve_release_binary(triple: &str) -> Result { + let version = env!("CARGO_PKG_VERSION"); + let cache_dir = dirs::cache_dir() + .unwrap_or_else(|| PathBuf::from("/tmp")) + .join("forger") + .join("builder-binaries"); + + let cached_path = cache_dir.join(format!("forger-{triple}-v{version}")); + + if cached_path.exists() { + info!(path = %cached_path.display(), "Using cached forger binary"); + return Ok(ResolvedBinary { path: cached_path }); + } + + let url = release_url(version, triple); + info!(%url, "Downloading forger binary for builder VM"); + + let response = reqwest::get(&url).await.map_err(|e| { + BuilderError::BinaryDownloadFailed { + url: url.clone(), + detail: e.to_string(), + } + })?; + + if !response.status().is_success() { + return Err(BuilderError::BinaryDownloadFailed { + url, + detail: format!("HTTP {}", response.status()), + }); + } + + let bytes = response.bytes().await.map_err(|e| { + BuilderError::BinaryDownloadFailed { + url: url.clone(), + detail: format!("reading response body: {e}"), + } + })?; + + std::fs::create_dir_all(&cache_dir).map_err(|e| BuilderError::BinaryDownloadFailed { + url: url.clone(), + detail: format!("creating cache dir: {e}"), + })?; + + std::fs::write(&cached_path, &bytes).map_err(|e| BuilderError::BinaryDownloadFailed { + url: url.clone(), + detail: format!("writing cached binary: {e}"), + })?; + + // Make executable + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let perms = std::fs::Permissions::from_mode(0o755); + std::fs::set_permissions(&cached_path, perms).map_err(|e| { + BuilderError::BinaryDownloadFailed { + url, + detail: format!("chmod: {e}"), + } + })?; + } + + Ok(ResolvedBinary { path: cached_path }) +} + +fn release_url(version: &str, triple: &str) -> String { + format!( + "https://github.com/CloudNebulaProject/refraction-forger/releases/download/v{version}/forger-{triple}" + ) +} + +/// Check if a path looks like a cross-compiled forger binary exists for this target. +pub fn dev_binary_path(triple: &str) -> Option { + let workspace_root = find_workspace_root()?; + let path = workspace_root + .join("target") + .join(triple) + .join("release") + .join("forger"); + path.exists().then_some(path) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn target_triple_mapping() { + assert_eq!(target_triple(&DistroFamily::OmniOS), "x86_64-unknown-illumos"); + assert_eq!(target_triple(&DistroFamily::Ubuntu), "x86_64-unknown-linux-gnu"); + } + + #[test] + fn release_url_construction() { + let url = release_url("0.1.0", "x86_64-unknown-linux-gnu"); + assert_eq!( + url, + "https://github.com/CloudNebulaProject/refraction-forger/releases/download/v0.1.0/forger-x86_64-unknown-linux-gnu" + ); + } + + #[test] + fn dev_detection_heuristic() { + // In test context, the binary is under target/ + let result = is_dev_build(); + // When running under `cargo test`, the binary IS in target/ + assert!(result); + } +} diff --git a/crates/forge-builder/src/config.rs b/crates/forge-builder/src/config.rs new file mode 100644 index 0000000..f14d450 --- /dev/null +++ b/crates/forge-builder/src/config.rs @@ -0,0 +1,89 @@ +use spec_parser::schema::{BuilderNode, DistroFamily}; + +/// Resolved builder VM configuration with defaults applied. +#[derive(Debug, Clone)] +pub struct BuilderConfig { + /// OCI reference, URL, or local path to the builder VM image. + pub image: String, + /// Number of virtual CPUs for the builder VM. + pub vcpus: u16, + /// Memory in MB for the builder VM. + pub memory_mb: u64, +} + +impl BuilderConfig { + /// Resolve a BuilderConfig from the optional spec node and distro family. + /// Spec values take precedence; convention defaults fill the rest. + pub fn resolve(spec_builder: Option<&BuilderNode>, distro: &DistroFamily) -> Self { + let default_image = Self::default_image(distro); + + match spec_builder { + Some(node) => Self { + image: node.image.clone().unwrap_or(default_image), + vcpus: node.vcpus.unwrap_or(2), + memory_mb: node.memory.unwrap_or(2048), + }, + None => Self { + image: default_image, + vcpus: 2, + memory_mb: 2048, + }, + } + } + + fn default_image(distro: &DistroFamily) -> String { + match distro { + DistroFamily::OmniOS => { + "oci://ghcr.io/cloudnebulaproject/omnios-builder:latest".to_string() + } + DistroFamily::Ubuntu => { + "oci://ghcr.io/cloudnebulaproject/ubuntu-builder:latest".to_string() + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_for_ubuntu() { + let config = BuilderConfig::resolve(None, &DistroFamily::Ubuntu); + assert!(config.image.contains("ubuntu-builder")); + assert_eq!(config.vcpus, 2); + assert_eq!(config.memory_mb, 2048); + } + + #[test] + fn defaults_for_omnios() { + let config = BuilderConfig::resolve(None, &DistroFamily::OmniOS); + assert!(config.image.contains("omnios-builder")); + } + + #[test] + fn spec_overrides_defaults() { + let node = BuilderNode { + image: Some("oci://custom/image:v1".to_string()), + vcpus: Some(4), + memory: Some(4096), + }; + let config = BuilderConfig::resolve(Some(&node), &DistroFamily::Ubuntu); + assert_eq!(config.image, "oci://custom/image:v1"); + assert_eq!(config.vcpus, 4); + assert_eq!(config.memory_mb, 4096); + } + + #[test] + fn partial_spec_fills_remaining_with_defaults() { + let node = BuilderNode { + image: None, + vcpus: Some(8), + memory: None, + }; + let config = BuilderConfig::resolve(Some(&node), &DistroFamily::Ubuntu); + assert!(config.image.contains("ubuntu-builder")); + assert_eq!(config.vcpus, 8); + assert_eq!(config.memory_mb, 2048); + } +} diff --git a/crates/forge-builder/src/detect.rs b/crates/forge-builder/src/detect.rs new file mode 100644 index 0000000..2152cf1 --- /dev/null +++ b/crates/forge-builder/src/detect.rs @@ -0,0 +1,92 @@ +use spec_parser::schema::{DistroFamily, ImageSpec, TargetKind}; + +/// Determine whether the current host needs a builder VM to build this spec. +/// +/// Returns `true` when: +/// - Any matching target is QCOW2 and we're not running as root (needs losetup/mount/chroot) +/// - The distro is OmniOS and we're not on illumos (needs pkg) +pub fn needs_builder(spec: &ImageSpec, target_name: Option<&str>, force_local: bool) -> bool { + if force_local { + return false; + } + + let distro = DistroFamily::from_distro_str(spec.distro.as_deref()); + let is_root = unsafe { libc::geteuid() == 0 }; + let is_illumos = cfg!(target_os = "illumos"); + + let has_qcow2 = spec.targets.iter().any(|t| { + let name_matches = target_name.is_none() || target_name == Some(t.name.as_str()); + name_matches && t.kind == TargetKind::Qcow2 + }); + + (has_qcow2 && !is_root) || (distro == DistroFamily::OmniOS && !is_illumos) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn make_spec(distro: Option<&str>, targets: &[(&str, &str)]) -> ImageSpec { + let kdl = { + let mut s = String::new(); + s.push_str(&format!( + "metadata name=\"test\" version=\"0.1.0\"\n" + )); + if let Some(d) = distro { + s.push_str(&format!("distro \"{d}\"\n")); + } + s.push_str("repositories {}\n"); + for (name, kind) in targets { + s.push_str(&format!("target \"{name}\" kind=\"{kind}\" {{\n")); + if *kind == "qcow2" { + s.push_str(" disk-size \"10G\"\n"); + } + s.push_str("}\n"); + } + s + }; + spec_parser::parse(&kdl).unwrap() + } + + #[test] + fn force_local_always_returns_false() { + let spec = make_spec(Some("ubuntu-22.04"), &[("vm", "qcow2")]); + assert!(!needs_builder(&spec, None, true)); + } + + #[test] + fn no_qcow2_targets_no_builder_needed() { + let spec = make_spec(Some("ubuntu-22.04"), &[("img", "oci")]); + // On Linux (not illumos), Ubuntu OCI targets don't need a builder + assert!(!needs_builder(&spec, None, false)); + } + + #[test] + fn qcow2_without_root_needs_builder() { + let spec = make_spec(Some("ubuntu-22.04"), &[("vm", "qcow2")]); + // This test is meaningful when run as non-root (CI, developer machines) + let is_root = unsafe { libc::geteuid() == 0 }; + assert_eq!(needs_builder(&spec, None, false), !is_root); + } + + #[test] + fn omnios_on_linux_needs_builder() { + let spec = make_spec(Some("omnios"), &[("img", "artifact")]); + let is_illumos = cfg!(target_os = "illumos"); + assert_eq!(needs_builder(&spec, None, false), !is_illumos); + } + + #[test] + fn target_filter_respected() { + let spec = make_spec( + Some("ubuntu-22.04"), + &[("vm", "qcow2"), ("img", "oci")], + ); + // When targeting only "img" (OCI), no qcow2 → no builder needed for that reason + let is_root = unsafe { libc::geteuid() == 0 }; + // Only the OmniOS check or root check matters; "img" is OCI so qcow2 check is false + assert_eq!(needs_builder(&spec, Some("img"), false), false || !is_root && false); + // Actually: has_qcow2 is false (target "img" is oci), distro is Ubuntu not OmniOS + assert!(!needs_builder(&spec, Some("img"), false)); + } +} diff --git a/crates/forge-builder/src/error.rs b/crates/forge-builder/src/error.rs new file mode 100644 index 0000000..1832b76 --- /dev/null +++ b/crates/forge-builder/src/error.rs @@ -0,0 +1,71 @@ +// See note in vm-manager/src/error.rs about thiserror 2 + edition 2024 +#![allow(unused_assignments)] + +use miette::Diagnostic; +use thiserror::Error; + +#[derive(Debug, Error, Diagnostic)] +pub enum BuilderError { + #[error("failed to resolve builder image: {detail}")] + #[diagnostic( + code(forge_builder::image_resolve_failed), + help("ensure the builder image reference is valid and the registry is reachable — for OCI images, check that GITHUB_TOKEN is set") + )] + ImageResolveFailed { detail: String }, + + #[error("builder VM lifecycle error at {phase}: {detail}")] + #[diagnostic( + code(forge_builder::vm_lifecycle), + help("ensure QEMU and KVM are available on this host — install qemu-system-x86_64 and check /dev/kvm permissions") + )] + VmLifecycle { phase: String, detail: String }, + + #[error("forger binary for target {target_triple} not found at {path}")] + #[diagnostic( + code(forge_builder::binary_not_found), + help("build the cross-compiled binary with:\n cargo build --target {target_triple} --release -p forger") + )] + BinaryNotFound { + target_triple: String, + path: String, + }, + + #[error("failed to download forger binary from {url}: {detail}")] + #[diagnostic( + code(forge_builder::binary_download_failed), + help("check network connectivity and that the release exists at the given URL") + )] + BinaryDownloadFailed { url: String, detail: String }, + + #[error("file transfer to builder VM failed: {detail}")] + #[diagnostic( + code(forge_builder::transfer_failed), + help("check that the builder VM is reachable via SSH and has enough disk space") + )] + TransferFailed { detail: String }, + + #[error("remote build inside builder VM failed with exit code {exit_code}")] + #[diagnostic( + code(forge_builder::remote_build_failed), + help("check the build output above for errors — the forger build ran inside the builder VM") + )] + RemoteBuildFailed { exit_code: i32 }, + + #[error("failed to download build artifacts from builder VM: {detail}")] + #[diagnostic( + code(forge_builder::download_failed), + help("the build may have succeeded but artifact retrieval failed — check VM connectivity") + )] + DownloadFailed { detail: String }, + + #[error("SSH keypair generation failed: {detail}")] + #[diagnostic( + code(forge_builder::keygen_failed), + help("this is an internal error in Ed25519 key generation — please report it") + )] + KeygenFailed { detail: String }, + + #[error(transparent)] + #[diagnostic(code(forge_builder::vm_error))] + VmError(#[from] vm_manager::VmError), +} diff --git a/crates/forge-builder/src/lib.rs b/crates/forge-builder/src/lib.rs new file mode 100644 index 0000000..0c57341 --- /dev/null +++ b/crates/forge-builder/src/lib.rs @@ -0,0 +1,96 @@ +pub mod binary; +pub mod config; +pub mod detect; +pub mod error; +pub mod lifecycle; +pub mod transfer; + +use std::io::{stderr, stdout}; +use std::path::Path; + +use spec_parser::schema::{DistroFamily, ImageSpec}; +use tracing::info; + +use crate::config::BuilderConfig; +use crate::error::BuilderError; + +/// Run a forger build inside a builder VM. +/// +/// This is the top-level orchestrator that: +/// 1. Resolves builder VM configuration from the spec +/// 2. Resolves the correct forger binary for the target OS +/// 3. Spins up an ephemeral builder VM +/// 4. Uploads inputs (binary, spec, files) +/// 5. Runs the build via SSH +/// 6. Downloads output artifacts +/// 7. Tears down the VM (always, even on error) +pub async fn run_in_builder( + spec: &ImageSpec, + spec_path: &Path, + files_dir: &Path, + output_dir: &Path, + target: Option<&str>, + profiles: &[String], +) -> Result<(), BuilderError> { + let distro = DistroFamily::from_distro_str(spec.distro.as_deref()); + let config = BuilderConfig::resolve(spec.builder.as_ref(), &distro); + let binary = binary::resolve_forger_binary(&distro).await?; + + info!("Starting builder VM for remote build"); + let session = lifecycle::BuilderSession::start(&config).await?; + + let result = run_build_in_session(&session, &binary.path, spec_path, files_dir, output_dir, target, profiles).await; + + // Always teardown, even on error + info!("Tearing down builder VM"); + if let Err(e) = session.teardown().await { + tracing::warn!(error = %e, "Builder VM teardown failed (build result preserved)"); + } + + result +} + +async fn run_build_in_session( + session: &lifecycle::BuilderSession, + binary_path: &Path, + spec_path: &Path, + files_dir: &Path, + output_dir: &Path, + target: Option<&str>, + profiles: &[String], +) -> Result<(), BuilderError> { + // Upload inputs + transfer::upload_build_inputs(session, binary_path, spec_path, files_dir)?; + + // Build the remote command + let mut cmd = String::from( + "sudo /tmp/forger-build/forger build -s /tmp/forger-build/spec.kdl -o /tmp/forger-build/output/ --local", + ); + + if let Some(t) = target { + cmd.push_str(&format!(" -t {t}")); + } + + for p in profiles { + cmd.push_str(&format!(" -p {p}")); + } + + info!(cmd = %cmd, "Running build in builder VM"); + + // Stream output to the user's terminal + let (_, _, exit_code) = + vm_manager::ssh::exec_streaming(&session.ssh_session, &cmd, stdout(), stderr()) + .map_err(|e| BuilderError::TransferFailed { + detail: format!("remote exec: {e}"), + })?; + + if exit_code != 0 { + return Err(BuilderError::RemoteBuildFailed { exit_code }); + } + + // Download artifacts + transfer::download_artifacts(session, output_dir)?; + + info!(output = %output_dir.display(), "Build artifacts downloaded successfully"); + Ok(()) +} diff --git a/crates/forge-builder/src/lifecycle.rs b/crates/forge-builder/src/lifecycle.rs new file mode 100644 index 0000000..059b965 --- /dev/null +++ b/crates/forge-builder/src/lifecycle.rs @@ -0,0 +1,194 @@ +use std::path::PathBuf; +use std::time::Duration; + +use ssh2::Session; +use tracing::info; + +use vm_manager::image::ImageManager; +use vm_manager::traits::Hypervisor; +use vm_manager::types::{CloudInitConfig, NetworkConfig, SshConfig, VmHandle, VmSpec}; +use vm_manager::RouterHypervisor; + +use crate::config::BuilderConfig; +use crate::error::BuilderError; + +/// An active builder VM session with SSH connectivity. +pub struct BuilderSession { + pub hypervisor: RouterHypervisor, + pub handle: VmHandle, + pub ssh_session: Session, + pub ssh_port: u16, +} + +impl BuilderSession { + /// Start a builder VM: resolve image, generate SSH keys, create + boot VM, connect SSH. + pub async fn start(config: &BuilderConfig) -> Result { + info!(image = %config.image, vcpus = config.vcpus, memory_mb = config.memory_mb, "Starting builder VM"); + + // 1. Resolve builder image + let image_path = resolve_builder_image(&config.image).await?; + + // 2. Generate ephemeral SSH keypair + let (pub_key, priv_pem) = generate_ssh_keypair()?; + + // 3. Build cloud-config with builder user + injected pubkey + let cloud_config = format!( + r#"#cloud-config +users: + - name: builder + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + ssh_authorized_keys: + - {pub_key} +"# + ); + + let ssh_config = SshConfig { + user: "builder".to_string(), + public_key: Some(pub_key), + private_key_path: None, + private_key_pem: Some(priv_pem), + }; + + // 4. Create VmSpec with user-mode networking (no root needed on host) + let vm_name = format!("forger-builder-{}", uuid_short()); + let spec = VmSpec { + name: vm_name.clone(), + image_path: image_path.clone(), + vcpus: config.vcpus, + memory_mb: config.memory_mb, + disk_gb: None, + network: NetworkConfig::User, + cloud_init: Some(CloudInitConfig { + user_data: cloud_config.into_bytes(), + instance_id: Some(vm_name.clone()), + hostname: Some("builder".to_string()), + }), + ssh: Some(ssh_config.clone()), + }; + + // 5. Prepare + start VM + let hypervisor = RouterHypervisor::new(None, None); + + let handle = hypervisor.prepare(&spec).await.map_err(|e| { + BuilderError::VmLifecycle { + phase: "prepare".into(), + detail: e.to_string(), + } + })?; + + let handle = hypervisor.start(&handle).await.map_err(|e| { + BuilderError::VmLifecycle { + phase: "start".into(), + detail: e.to_string(), + } + })?; + + // 6. Connect SSH with retry (user-mode networking uses host port forwarding) + let ssh_port = handle.ssh_host_port.unwrap_or(22); + let ssh_ip = "127.0.0.1"; + + info!(port = ssh_port, "Waiting for SSH connection to builder VM"); + + let ssh_session = vm_manager::ssh::connect_with_retry( + ssh_ip, + ssh_port, + &ssh_config, + Duration::from_secs(120), + ) + .await + .map_err(|e| BuilderError::VmLifecycle { + phase: "ssh_connect".into(), + detail: e.to_string(), + })?; + + info!("Builder VM ready"); + + Ok(Self { + hypervisor, + handle, + ssh_session, + ssh_port, + }) + } + + /// Tear down the builder VM, destroying all resources. + pub async fn teardown(self) -> Result<(), BuilderError> { + info!(name = %self.handle.name, "Tearing down builder VM"); + + // Drop SSH session first + drop(self.ssh_session); + + self.hypervisor + .destroy(self.handle) + .await + .map_err(|e| BuilderError::VmLifecycle { + phase: "destroy".into(), + detail: e.to_string(), + })?; + + Ok(()) + } +} + +/// Resolve builder image from OCI reference, URL, or local path. +async fn resolve_builder_image(image: &str) -> Result { + let mgr = ImageManager::new(); + + if image.starts_with("oci://") { + let reference = image.strip_prefix("oci://").unwrap(); + mgr.pull_oci(reference, None) + .await + .map_err(|e| BuilderError::ImageResolveFailed { + detail: format!("OCI pull {reference}: {e}"), + }) + } else if image.starts_with("http://") || image.starts_with("https://") { + mgr.pull(image, None) + .await + .map_err(|e| BuilderError::ImageResolveFailed { + detail: format!("download {image}: {e}"), + }) + } else { + // Local path + let path = PathBuf::from(image); + if !path.exists() { + return Err(BuilderError::ImageResolveFailed { + detail: format!("local image not found: {}", path.display()), + }); + } + Ok(path) + } +} + +fn generate_ssh_keypair() -> Result<(String, String), BuilderError> { + use ssh_key::{Algorithm, LineEnding, PrivateKey, rand_core::OsRng}; + + let sk = PrivateKey::random(&mut OsRng, Algorithm::Ed25519).map_err(|e| { + BuilderError::KeygenFailed { + detail: format!("Ed25519 key generation: {e}"), + } + })?; + + let pub_openssh = sk.public_key().to_openssh().map_err(|e| { + BuilderError::KeygenFailed { + detail: format!("serialize public key: {e}"), + } + })?; + + let priv_pem = sk.to_openssh(LineEnding::LF).map_err(|e| { + BuilderError::KeygenFailed { + detail: format!("serialize private key: {e}"), + } + })?; + + Ok((pub_openssh, priv_pem.to_string())) +} + +fn uuid_short() -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + format!("{:x}", ts & 0xFFFF_FFFF) +} diff --git a/crates/forge-builder/src/transfer.rs b/crates/forge-builder/src/transfer.rs new file mode 100644 index 0000000..78184a7 --- /dev/null +++ b/crates/forge-builder/src/transfer.rs @@ -0,0 +1,149 @@ +use std::path::{Path, PathBuf}; + +use tracing::info; + +use vm_manager::ssh; + +use crate::error::BuilderError; +use crate::lifecycle::BuilderSession; + +const REMOTE_BUILD_DIR: &str = "/tmp/forger-build"; + +/// Upload all build inputs to the builder VM. +pub fn upload_build_inputs( + session: &BuilderSession, + forger_binary: &Path, + spec_path: &Path, + files_dir: &Path, +) -> Result<(), BuilderError> { + let sess = &session.ssh_session; + + // Create remote build directory + ssh::exec(sess, &format!("mkdir -p {REMOTE_BUILD_DIR}/output")) + .map_err(|e| BuilderError::TransferFailed { + detail: format!("mkdir: {e}"), + })?; + + // Upload forger binary + info!("Uploading forger binary to builder VM"); + let remote_forger = PathBuf::from(format!("{REMOTE_BUILD_DIR}/forger")); + ssh::upload(sess, forger_binary, &remote_forger).map_err(|e| { + BuilderError::TransferFailed { + detail: format!("upload forger binary: {e}"), + } + })?; + + // Make executable + ssh::exec(sess, &format!("chmod +x {REMOTE_BUILD_DIR}/forger")).map_err(|e| { + BuilderError::TransferFailed { + detail: format!("chmod forger: {e}"), + } + })?; + + // Upload spec file + info!("Uploading spec file"); + let remote_spec = PathBuf::from(format!("{REMOTE_BUILD_DIR}/spec.kdl")); + ssh::upload(sess, spec_path, &remote_spec).map_err(|e| BuilderError::TransferFailed { + detail: format!("upload spec: {e}"), + })?; + + // Upload files/ directory if it exists (tar locally → upload → extract remotely) + if files_dir.exists() && files_dir.is_dir() { + upload_directory(sess, files_dir, &format!("{REMOTE_BUILD_DIR}/files"))?; + } + + Ok(()) +} + +/// Upload a local directory to the VM by creating a tar, uploading, and extracting. +fn upload_directory( + sess: &ssh2::Session, + local_dir: &Path, + remote_dir: &str, +) -> Result<(), BuilderError> { + info!(local = %local_dir.display(), remote = %remote_dir, "Uploading directory to builder VM"); + + // Create a tar archive in memory + let mut tar_buf = Vec::new(); + { + let mut ar = tar::Builder::new(&mut tar_buf); + ar.append_dir_all(".", local_dir) + .map_err(|e| BuilderError::TransferFailed { + detail: format!("tar {}: {e}", local_dir.display()), + })?; + ar.finish().map_err(|e| BuilderError::TransferFailed { + detail: format!("tar finish: {e}"), + })?; + } + + // Write tar to a temp file so we can upload it + let tmp = tempfile::NamedTempFile::new().map_err(|e| BuilderError::TransferFailed { + detail: format!("tempfile: {e}"), + })?; + std::fs::write(tmp.path(), &tar_buf).map_err(|e| BuilderError::TransferFailed { + detail: format!("write tar: {e}"), + })?; + + let remote_tar = PathBuf::from(format!("{remote_dir}.tar")); + ssh::upload(sess, tmp.path(), &remote_tar).map_err(|e| BuilderError::TransferFailed { + detail: format!("upload tar: {e}"), + })?; + + // Extract on remote + ssh::exec( + sess, + &format!("mkdir -p {remote_dir} && tar xf {remote_dir}.tar -C {remote_dir} && rm {remote_dir}.tar"), + ) + .map_err(|e| BuilderError::TransferFailed { + detail: format!("extract tar: {e}"), + })?; + + Ok(()) +} + +/// Download build artifacts from the builder VM. +pub fn download_artifacts( + session: &BuilderSession, + output_dir: &Path, +) -> Result<(), BuilderError> { + let sess = &session.ssh_session; + let remote_output = format!("{REMOTE_BUILD_DIR}/output"); + + // List files in remote output directory + let (stdout, _, exit_code) = ssh::exec( + sess, + &format!("find {remote_output} -maxdepth 1 -type f -printf '%f\\n'"), + ) + .map_err(|e| BuilderError::DownloadFailed { + detail: format!("list remote files: {e}"), + })?; + + if exit_code != 0 { + return Err(BuilderError::DownloadFailed { + detail: "failed to list remote output directory".to_string(), + }); + } + + std::fs::create_dir_all(output_dir).map_err(|e| BuilderError::DownloadFailed { + detail: format!("create output dir: {e}"), + })?; + + for filename in stdout.lines() { + let filename = filename.trim(); + if filename.is_empty() { + continue; + } + + let remote_path = PathBuf::from(format!("{remote_output}/{filename}")); + let local_path = output_dir.join(filename); + + info!(file = %filename, "Downloading artifact from builder VM"); + ssh::download(sess, &remote_path, &local_path).map_err(|e| { + BuilderError::DownloadFailed { + detail: format!("download {filename}: {e}"), + } + })?; + } + + Ok(()) +} diff --git a/crates/forge-engine/src/phase1/mod.rs b/crates/forge-engine/src/phase1/mod.rs index 7f209c7..bb46369 100644 --- a/crates/forge-engine/src/phase1/mod.rs +++ b/crates/forge-engine/src/phase1/mod.rs @@ -119,15 +119,23 @@ async fn execute_apt( let mirror_url = first_mirror .map(|m| m.url.as_str()) .unwrap_or("http://archive.ubuntu.com/ubuntu"); + let components = first_mirror.and_then(|m| m.components.as_deref()); // Bootstrap the rootfs - crate::tools::apt::debootstrap(runner, suite, root, mirror_url).await?; + crate::tools::apt::debootstrap(runner, suite, root, mirror_url, components).await?; - // Add any additional APT mirror sources (skip the first one used for debootstrap) - for mirror in spec.repositories.apt_mirrors.iter().skip(1) { - let components = mirror.components.as_deref().unwrap_or("main"); - let entry = format!("deb {} {} {}", mirror.url, mirror.suite, components); - crate::tools::apt::add_source(runner, root, &entry).await?; + // Write sources.list with full component lists from all apt-mirror entries + let source_entries: Vec = spec + .repositories + .apt_mirrors + .iter() + .map(|m| { + let components = m.components.as_deref().unwrap_or("main"); + format!("deb {} {} {}", m.url, m.suite, components) + }) + .collect(); + if !source_entries.is_empty() { + crate::tools::apt::write_sources_list(root, &source_entries).await?; } // Update package lists diff --git a/crates/forge-engine/src/phase2/qcow2_ext4.rs b/crates/forge-engine/src/phase2/qcow2_ext4.rs index c1820b0..eec3602 100644 --- a/crates/forge-engine/src/phase2/qcow2_ext4.rs +++ b/crates/forge-engine/src/phase2/qcow2_ext4.rs @@ -90,7 +90,7 @@ pub async fn build_qcow2_ext4( "chroot", &[ mount_str, - "grub-install", + "/usr/sbin/grub-install", "--target=x86_64-efi", "--efi-directory=/boot/efi", "--no-nvram", @@ -102,7 +102,7 @@ pub async fn build_qcow2_ext4( runner .run( "chroot", - &[mount_str, "grub-mkconfig", "-o", "/boot/grub/grub.cfg"], + &[mount_str, "/usr/sbin/grub-mkconfig", "-o", "/boot/grub/grub.cfg"], ) .await?; diff --git a/crates/forge-engine/src/tools/apt.rs b/crates/forge-engine/src/tools/apt.rs index b094498..768a472 100644 --- a/crates/forge-engine/src/tools/apt.rs +++ b/crates/forge-engine/src/tools/apt.rs @@ -10,14 +10,18 @@ pub async fn debootstrap( suite: &str, root: &str, mirror: &str, + components: Option<&str>, ) -> Result<(), ForgeError> { - info!(suite, root, mirror, "Running debootstrap"); - runner - .run( - "debootstrap", - &["--arch", "amd64", suite, root, mirror], - ) - .await?; + info!(suite, root, mirror, ?components, "Running debootstrap"); + let mut args = vec!["--arch", "amd64"]; + // Format: --components=main,universe (comma-separated) + let comp_arg; + if let Some(c) = components { + comp_arg = format!("--components={}", c.replace(' ', ",")); + args.push(&comp_arg); + } + args.extend_from_slice(&[suite, root, mirror]); + runner.run("debootstrap", &args).await?; Ok(()) } @@ -47,6 +51,22 @@ pub async fn install( Ok(()) } +/// Write the primary sources.list in the chroot, replacing the debootstrap default. +pub async fn write_sources_list( + root: &str, + entries: &[String], +) -> Result<(), ForgeError> { + let sources_path = Path::new(root).join("etc/apt/sources.list"); + info!(?sources_path, count = entries.len(), "Writing sources.list"); + let content = entries.join("\n") + "\n"; + std::fs::write(&sources_path, content).map_err(|e| ForgeError::Overlay { + action: "write sources.list".to_string(), + detail: sources_path.display().to_string(), + source: e, + })?; + Ok(()) +} + /// Add an APT source entry to the chroot's sources.list.d/. pub async fn add_source( runner: &dyn ToolRunner, @@ -112,7 +132,7 @@ mod tests { #[tokio::test] async fn test_debootstrap_args() { let runner = MockToolRunner::new(); - debootstrap(&runner, "jammy", "/tmp/root", "http://archive.ubuntu.com/ubuntu") + debootstrap(&runner, "jammy", "/tmp/root", "http://archive.ubuntu.com/ubuntu", None) .await .unwrap(); @@ -125,6 +145,22 @@ mod tests { ); } + #[tokio::test] + async fn test_debootstrap_with_components() { + let runner = MockToolRunner::new(); + debootstrap(&runner, "jammy", "/tmp/root", "http://archive.ubuntu.com/ubuntu", Some("main universe")) + .await + .unwrap(); + + let calls = runner.calls(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].0, "debootstrap"); + assert_eq!( + calls[0].1, + vec!["--arch", "amd64", "--components=main,universe", "jammy", "/tmp/root", "http://archive.ubuntu.com/ubuntu"] + ); + } + #[tokio::test] async fn test_install_args() { let runner = MockToolRunner::new(); diff --git a/crates/forger/Cargo.toml b/crates/forger/Cargo.toml index 6a561d6..3dca739 100644 --- a/crates/forger/Cargo.toml +++ b/crates/forger/Cargo.toml @@ -4,10 +4,15 @@ version = "0.1.0" edition.workspace = true rust-version.workspace = true +[features] +default = ["builder"] +builder = ["dep:forge-builder"] + [dependencies] spec-parser = { workspace = true } forge-oci = { workspace = true } forge-engine = { workspace = true } +forge-builder = { workspace = true, optional = true } clap = { workspace = true } miette = { workspace = true } thiserror = { workspace = true } diff --git a/crates/forger/src/commands/build.rs b/crates/forger/src/commands/build.rs index 522e18d..3732825 100644 --- a/crates/forger/src/commands/build.rs +++ b/crates/forger/src/commands/build.rs @@ -11,6 +11,8 @@ pub async fn run( target: Option<&str>, profiles: &[String], output_dir: &PathBuf, + local: bool, + use_builder: bool, ) -> miette::Result<()> { let kdl_content = std::fs::read_to_string(spec_path) .into_diagnostic() @@ -33,6 +35,35 @@ pub async fn run( // Determine files directory (images/files/ relative to spec) let files_dir = spec_dir.join("files"); + // Check if we need a builder VM + #[cfg(feature = "builder")] + { + let needs = forge_builder::detect::needs_builder(&filtered, target, local); + if needs || use_builder { + info!("Delegating build to builder VM"); + forge_builder::run_in_builder( + &filtered, + spec_path, + &files_dir, + output_dir, + target, + profiles, + ) + .await + .map_err(miette::Report::new) + .wrap_err("Builder VM build failed")?; + + println!("Build complete. Output: {}", output_dir.display()); + return Ok(()); + } + } + + // Suppress unused variable warnings when builder feature is disabled + #[cfg(not(feature = "builder"))] + { + let _ = (local, use_builder); + } + let runner = SystemToolRunner; let ctx = BuildContext { diff --git a/crates/forger/src/main.rs b/crates/forger/src/main.rs index 5eccf5b..7ebcf5b 100644 --- a/crates/forger/src/main.rs +++ b/crates/forger/src/main.rs @@ -36,6 +36,14 @@ enum Commands { /// Output directory for build artifacts #[arg(short, long, default_value = "./output")] output_dir: PathBuf, + + /// Force local build (skip builder VM detection) + #[arg(long)] + local: bool, + + /// Force build inside a builder VM + #[arg(long, conflicts_with = "local")] + use_builder: bool, }, /// Validate a spec file (parse + resolve includes) @@ -99,8 +107,10 @@ async fn main() -> Result<()> { target, profile, output_dir, + local, + use_builder, } => { - commands::build::run(&spec, target.as_deref(), &profile, &output_dir).await?; + commands::build::run(&spec, target.as_deref(), &profile, &output_dir, local, use_builder).await?; } Commands::Validate { spec } => { commands::validate::run(&spec)?; diff --git a/crates/spec-parser/src/lib.rs b/crates/spec-parser/src/lib.rs index ff489fc..6917a52 100644 --- a/crates/spec-parser/src/lib.rs +++ b/crates/spec-parser/src/lib.rs @@ -239,4 +239,67 @@ mod tests { assert_eq!(pool.properties[0].name, "ashift"); assert_eq!(pool.properties[0].value, "12"); } + + #[test] + fn test_parse_builder_node_full() { + let kdl = r#" + metadata name="test" version="0.1.0" + repositories {} + builder { + image "oci://ghcr.io/custom/builder:v1" + vcpus 4 + memory 4096 + } + "#; + + let spec = parse(kdl).expect("Failed to parse KDL"); + let builder = spec.builder.as_ref().unwrap(); + assert_eq!(builder.image.as_deref(), Some("oci://ghcr.io/custom/builder:v1")); + assert_eq!(builder.vcpus, Some(4)); + assert_eq!(builder.memory, Some(4096)); + } + + #[test] + fn test_parse_builder_node_partial() { + let kdl = r#" + metadata name="test" version="0.1.0" + repositories {} + builder { + vcpus 8 + } + "#; + + let spec = parse(kdl).expect("Failed to parse KDL"); + let builder = spec.builder.as_ref().unwrap(); + assert_eq!(builder.image, None); + assert_eq!(builder.vcpus, Some(8)); + assert_eq!(builder.memory, None); + } + + #[test] + fn test_parse_builder_node_empty() { + let kdl = r#" + metadata name="test" version="0.1.0" + repositories {} + builder { + } + "#; + + let spec = parse(kdl).expect("Failed to parse KDL"); + let builder = spec.builder.as_ref().unwrap(); + assert_eq!(builder.image, None); + assert_eq!(builder.vcpus, None); + assert_eq!(builder.memory, None); + } + + #[test] + fn test_parse_no_builder_node() { + let kdl = r#" + metadata name="test" version="0.1.0" + repositories {} + "#; + + let spec = parse(kdl).expect("Failed to parse KDL"); + assert!(spec.builder.is_none()); + } } diff --git a/crates/spec-parser/src/resolve.rs b/crates/spec-parser/src/resolve.rs index 19a6b6c..c2a296c 100644 --- a/crates/spec-parser/src/resolve.rs +++ b/crates/spec-parser/src/resolve.rs @@ -210,6 +210,11 @@ fn merge_base(mut base: ImageSpec, child: ImageSpec) -> ImageSpec { base.targets = child.targets; } + // builder: child's builder replaces base entirely + if child.builder.is_some() { + base.builder = child.builder; + } + base } diff --git a/crates/spec-parser/src/schema.rs b/crates/spec-parser/src/schema.rs index 6b01473..e0c36d2 100644 --- a/crates/spec-parser/src/schema.rs +++ b/crates/spec-parser/src/schema.rs @@ -58,6 +58,22 @@ pub struct ImageSpec { #[knuffel(children(name = "target"))] pub targets: Vec, + + #[knuffel(child)] + pub builder: Option, +} + +/// Configuration for a builder VM used when the host can't build locally. +#[derive(Debug, Decode)] +pub struct BuilderNode { + #[knuffel(child, unwrap(argument))] + pub image: Option, + + #[knuffel(child, unwrap(argument))] + pub vcpus: Option, + + #[knuffel(child, unwrap(argument))] + pub memory: Option, } #[derive(Debug, Decode)]