mirror of
https://github.com/CloudNebulaProject/refraction-forger.git
synced 2026-04-10 21:30:40 +00:00
Add host-side OCI push and builder image override for bootstrap builds
Move OCI push from builder VM (where GITHUB_TOKEN is unavailable) to the host side. Add --skip-push and --builder-image CLI flags so the build pipeline can be bootstrapped before builder OCI images exist. KDL specs now include public cloud image URLs as default builder images. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
7d97061e0f
commit
f880889589
10 changed files with 132 additions and 8 deletions
1
Cargo.lock
generated
1
Cargo.lock
generated
|
|
@ -715,6 +715,7 @@ name = "forge-builder"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"dirs",
|
"dirs",
|
||||||
|
"forge-oci",
|
||||||
"libc",
|
"libc",
|
||||||
"miette 7.6.0",
|
"miette 7.6.0",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,7 @@ rust-version.workspace = true
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
spec-parser = { workspace = true }
|
spec-parser = { workspace = true }
|
||||||
|
forge-oci = { workspace = true }
|
||||||
vm-manager = { workspace = true }
|
vm-manager = { workspace = true }
|
||||||
miette = { workspace = true }
|
miette = { workspace = true }
|
||||||
thiserror = { workspace = true }
|
thiserror = { workspace = true }
|
||||||
|
|
|
||||||
|
|
@ -65,6 +65,13 @@ pub enum BuilderError {
|
||||||
)]
|
)]
|
||||||
KeygenFailed { detail: String },
|
KeygenFailed { detail: String },
|
||||||
|
|
||||||
|
#[error("failed to push artifact to OCI registry: {detail}")]
|
||||||
|
#[diagnostic(
|
||||||
|
code(forge_builder::artifact_push_failed),
|
||||||
|
help("check that GITHUB_TOKEN is set and the registry reference is valid")
|
||||||
|
)]
|
||||||
|
ArtifactPushFailed { detail: String },
|
||||||
|
|
||||||
#[error(transparent)]
|
#[error(transparent)]
|
||||||
#[diagnostic(code(forge_builder::vm_error))]
|
#[diagnostic(code(forge_builder::vm_error))]
|
||||||
VmError(#[from] vm_manager::VmError),
|
VmError(#[from] vm_manager::VmError),
|
||||||
|
|
|
||||||
|
|
@ -3,6 +3,7 @@ pub mod config;
|
||||||
pub mod detect;
|
pub mod detect;
|
||||||
pub mod error;
|
pub mod error;
|
||||||
pub mod lifecycle;
|
pub mod lifecycle;
|
||||||
|
pub mod push;
|
||||||
pub mod transfer;
|
pub mod transfer;
|
||||||
|
|
||||||
use std::io::{stderr, stdout};
|
use std::io::{stderr, stdout};
|
||||||
|
|
@ -31,15 +32,23 @@ pub async fn run_in_builder(
|
||||||
output_dir: &Path,
|
output_dir: &Path,
|
||||||
target: Option<&str>,
|
target: Option<&str>,
|
||||||
profiles: &[String],
|
profiles: &[String],
|
||||||
|
builder_image: Option<&str>,
|
||||||
|
skip_push: bool,
|
||||||
) -> Result<(), BuilderError> {
|
) -> Result<(), BuilderError> {
|
||||||
let distro = DistroFamily::from_distro_str(spec.distro.as_deref());
|
let distro = DistroFamily::from_distro_str(spec.distro.as_deref());
|
||||||
let config = BuilderConfig::resolve(spec.builder.as_ref(), &distro);
|
let mut config = BuilderConfig::resolve(spec.builder.as_ref(), &distro);
|
||||||
|
|
||||||
|
// CLI --builder-image overrides the spec/default image
|
||||||
|
if let Some(img) = builder_image {
|
||||||
|
config.image = img.to_string();
|
||||||
|
}
|
||||||
|
|
||||||
let binary = binary::resolve_forger_binary(&distro).await?;
|
let binary = binary::resolve_forger_binary(&distro).await?;
|
||||||
|
|
||||||
info!("Starting builder VM for remote build");
|
info!("Starting builder VM for remote build");
|
||||||
let session = lifecycle::BuilderSession::start(&config).await?;
|
let session = lifecycle::BuilderSession::start(&config).await?;
|
||||||
|
|
||||||
let result = run_build_in_session(&session, spec, &binary.path, spec_path, files_dir, output_dir, target, profiles).await;
|
let result = run_build_in_session(&session, spec, &binary.path, spec_path, files_dir, output_dir, target, profiles, skip_push).await;
|
||||||
|
|
||||||
// On failure, try to collect diagnostic info before teardown
|
// On failure, try to collect diagnostic info before teardown
|
||||||
if let Err(ref e) = result {
|
if let Err(ref e) = result {
|
||||||
|
|
@ -180,6 +189,7 @@ async fn run_build_in_session(
|
||||||
output_dir: &Path,
|
output_dir: &Path,
|
||||||
target: Option<&str>,
|
target: Option<&str>,
|
||||||
profiles: &[String],
|
profiles: &[String],
|
||||||
|
skip_push: bool,
|
||||||
) -> Result<(), BuilderError> {
|
) -> Result<(), BuilderError> {
|
||||||
// Verify network connectivity (DNS) before doing anything
|
// Verify network connectivity (DNS) before doing anything
|
||||||
verify_network(session)?;
|
verify_network(session)?;
|
||||||
|
|
@ -190,9 +200,10 @@ async fn run_build_in_session(
|
||||||
// Upload inputs
|
// Upload inputs
|
||||||
transfer::upload_build_inputs(session, binary_path, spec_path, files_dir)?;
|
transfer::upload_build_inputs(session, binary_path, spec_path, files_dir)?;
|
||||||
|
|
||||||
// Build the remote command
|
// Build the remote command — always pass --skip-push so the VM never attempts
|
||||||
|
// to push to the registry (it lacks GITHUB_TOKEN); the host handles pushing.
|
||||||
let mut cmd = String::from(
|
let mut cmd = String::from(
|
||||||
"sudo /tmp/forger-build/forger build -s /tmp/forger-build/spec.kdl -o /tmp/forger-build/output/ --local",
|
"sudo /tmp/forger-build/forger build -s /tmp/forger-build/spec.kdl -o /tmp/forger-build/output/ --local --skip-push",
|
||||||
);
|
);
|
||||||
|
|
||||||
if let Some(t) = target {
|
if let Some(t) = target {
|
||||||
|
|
@ -230,5 +241,11 @@ async fn run_build_in_session(
|
||||||
transfer::download_artifacts(session, output_dir)?;
|
transfer::download_artifacts(session, output_dir)?;
|
||||||
|
|
||||||
info!(output = %output_dir.display(), "Build artifacts downloaded successfully");
|
info!(output = %output_dir.display(), "Build artifacts downloaded successfully");
|
||||||
|
|
||||||
|
// Host-side push: push QCOW2 outputs from the host where GITHUB_TOKEN is available
|
||||||
|
if !skip_push {
|
||||||
|
push::push_qcow2_outputs(spec, output_dir).await?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
62
crates/forge-builder/src/push.rs
Normal file
62
crates/forge-builder/src/push.rs
Normal file
|
|
@ -0,0 +1,62 @@
|
||||||
|
use std::path::Path;
|
||||||
|
|
||||||
|
use spec_parser::schema::ImageSpec;
|
||||||
|
use tracing::info;
|
||||||
|
|
||||||
|
use crate::error::BuilderError;
|
||||||
|
|
||||||
|
/// Push QCOW2 outputs to OCI registries from the host side.
|
||||||
|
///
|
||||||
|
/// Iterates over spec targets, finds QCOW2 files with `push_to` set,
|
||||||
|
/// and pushes them via `forge_oci::artifact`. This mirrors
|
||||||
|
/// `forge_engine::phase2::push_qcow2_if_configured()` but runs on the
|
||||||
|
/// host where `GITHUB_TOKEN` is available.
|
||||||
|
pub async fn push_qcow2_outputs(spec: &ImageSpec, output_dir: &Path) -> Result<(), BuilderError> {
|
||||||
|
for target in &spec.targets {
|
||||||
|
let Some(ref push_ref) = target.push_to else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
|
let qcow2_path = output_dir.join(format!("{}.qcow2", target.name));
|
||||||
|
|
||||||
|
if !qcow2_path.exists() {
|
||||||
|
info!(
|
||||||
|
target = %target.name,
|
||||||
|
path = %qcow2_path.display(),
|
||||||
|
"QCOW2 file not found — skipping push"
|
||||||
|
);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
info!(
|
||||||
|
reference = %push_ref,
|
||||||
|
path = %qcow2_path.display(),
|
||||||
|
"Pushing QCOW2 artifact to OCI registry (host-side)"
|
||||||
|
);
|
||||||
|
|
||||||
|
let qcow2_data =
|
||||||
|
std::fs::read(&qcow2_path).map_err(|e| BuilderError::ArtifactPushFailed {
|
||||||
|
detail: format!("failed to read QCOW2 file {}: {e}", qcow2_path.display()),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
let metadata = forge_oci::artifact::Qcow2Metadata {
|
||||||
|
name: target.name.clone(),
|
||||||
|
version: "latest".to_string(),
|
||||||
|
architecture: "amd64".to_string(),
|
||||||
|
os: "linux".to_string(),
|
||||||
|
description: None,
|
||||||
|
};
|
||||||
|
|
||||||
|
let auth = forge_oci::artifact::resolve_ghcr_auth();
|
||||||
|
|
||||||
|
forge_oci::artifact::push_qcow2_artifact(push_ref, qcow2_data, &metadata, &auth, &[])
|
||||||
|
.await
|
||||||
|
.map_err(|e| BuilderError::ArtifactPushFailed {
|
||||||
|
detail: e.to_string(),
|
||||||
|
})?;
|
||||||
|
|
||||||
|
info!(reference = %push_ref, "Host-side push complete");
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
@ -23,6 +23,8 @@ pub struct BuildContext<'a> {
|
||||||
pub output_dir: &'a Path,
|
pub output_dir: &'a Path,
|
||||||
/// Tool runner for executing external commands.
|
/// Tool runner for executing external commands.
|
||||||
pub runner: &'a dyn ToolRunner,
|
pub runner: &'a dyn ToolRunner,
|
||||||
|
/// Skip OCI registry push after build (host-side push handles it instead).
|
||||||
|
pub skip_push: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> BuildContext<'a> {
|
impl<'a> BuildContext<'a> {
|
||||||
|
|
@ -88,8 +90,10 @@ impl<'a> BuildContext<'a> {
|
||||||
finalize_result?;
|
finalize_result?;
|
||||||
cleanup_result?;
|
cleanup_result?;
|
||||||
|
|
||||||
// Auto-push to OCI registry if configured
|
// Auto-push to OCI registry if configured (skipped when host-side push handles it)
|
||||||
|
if !self.skip_push {
|
||||||
phase2::push_qcow2_if_configured(target, self.output_dir).await?;
|
phase2::push_qcow2_if_configured(target, self.output_dir).await?;
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -13,6 +13,8 @@ pub async fn run(
|
||||||
output_dir: &PathBuf,
|
output_dir: &PathBuf,
|
||||||
local: bool,
|
local: bool,
|
||||||
use_builder: bool,
|
use_builder: bool,
|
||||||
|
skip_push: bool,
|
||||||
|
builder_image: Option<&str>,
|
||||||
) -> miette::Result<()> {
|
) -> miette::Result<()> {
|
||||||
let kdl_content = std::fs::read_to_string(spec_path)
|
let kdl_content = std::fs::read_to_string(spec_path)
|
||||||
.into_diagnostic()
|
.into_diagnostic()
|
||||||
|
|
@ -48,6 +50,8 @@ pub async fn run(
|
||||||
output_dir,
|
output_dir,
|
||||||
target,
|
target,
|
||||||
profiles,
|
profiles,
|
||||||
|
builder_image,
|
||||||
|
skip_push,
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.map_err(miette::Report::new)
|
.map_err(miette::Report::new)
|
||||||
|
|
@ -61,7 +65,7 @@ pub async fn run(
|
||||||
// Suppress unused variable warnings when builder feature is disabled
|
// Suppress unused variable warnings when builder feature is disabled
|
||||||
#[cfg(not(feature = "builder"))]
|
#[cfg(not(feature = "builder"))]
|
||||||
{
|
{
|
||||||
let _ = (local, use_builder);
|
let _ = (local, use_builder, builder_image);
|
||||||
}
|
}
|
||||||
|
|
||||||
let runner = SystemToolRunner;
|
let runner = SystemToolRunner;
|
||||||
|
|
@ -71,6 +75,7 @@ pub async fn run(
|
||||||
files_dir: &files_dir,
|
files_dir: &files_dir,
|
||||||
output_dir,
|
output_dir,
|
||||||
runner: &runner,
|
runner: &runner,
|
||||||
|
skip_push,
|
||||||
};
|
};
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
|
|
|
||||||
|
|
@ -44,6 +44,14 @@ enum Commands {
|
||||||
/// Force build inside a builder VM
|
/// Force build inside a builder VM
|
||||||
#[arg(long, conflicts_with = "local")]
|
#[arg(long, conflicts_with = "local")]
|
||||||
use_builder: bool,
|
use_builder: bool,
|
||||||
|
|
||||||
|
/// Skip OCI registry push (host-side push used with builder VMs)
|
||||||
|
#[arg(long)]
|
||||||
|
skip_push: bool,
|
||||||
|
|
||||||
|
/// Override builder VM image (path, URL, or oci:// reference)
|
||||||
|
#[arg(long)]
|
||||||
|
builder_image: Option<String>,
|
||||||
},
|
},
|
||||||
|
|
||||||
/// Validate a spec file (parse + resolve includes)
|
/// Validate a spec file (parse + resolve includes)
|
||||||
|
|
@ -109,8 +117,20 @@ async fn main() -> Result<()> {
|
||||||
output_dir,
|
output_dir,
|
||||||
local,
|
local,
|
||||||
use_builder,
|
use_builder,
|
||||||
|
skip_push,
|
||||||
|
builder_image,
|
||||||
} => {
|
} => {
|
||||||
commands::build::run(&spec, target.as_deref(), &profile, &output_dir, local, use_builder).await?;
|
commands::build::run(
|
||||||
|
&spec,
|
||||||
|
target.as_deref(),
|
||||||
|
&profile,
|
||||||
|
&output_dir,
|
||||||
|
local,
|
||||||
|
use_builder,
|
||||||
|
skip_push,
|
||||||
|
builder_image.as_deref(),
|
||||||
|
)
|
||||||
|
.await?;
|
||||||
}
|
}
|
||||||
Commands::Validate { spec } => {
|
Commands::Validate { spec } => {
|
||||||
commands::validate::run(&spec)?;
|
commands::validate::run(&spec)?;
|
||||||
|
|
|
||||||
|
|
@ -22,6 +22,12 @@ overlays {
|
||||||
file destination="/etc/default/init" source="default_init.utc" owner="root" group="root" mode="644"
|
file destination="/etc/default/init" source="default_init.utc" owner="root" group="root" mode="644"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
builder {
|
||||||
|
image "https://downloads.omnios.org/media/bloody/omnios-bloody-cloud.raw.zst"
|
||||||
|
vcpus 4
|
||||||
|
memory 4096
|
||||||
|
}
|
||||||
|
|
||||||
target "qcow2" kind="qcow2" {
|
target "qcow2" kind="qcow2" {
|
||||||
disk-size "4000M"
|
disk-size "4000M"
|
||||||
bootloader "uefi"
|
bootloader "uefi"
|
||||||
|
|
|
||||||
|
|
@ -31,6 +31,7 @@ overlays {
|
||||||
}
|
}
|
||||||
|
|
||||||
builder {
|
builder {
|
||||||
|
image "https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img"
|
||||||
vcpus 4
|
vcpus 4
|
||||||
memory 4096
|
memory 4096
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue