solstice-ci/.mise/tasks/ci/vm-build

#!/usr/bin/env bash
set -euo pipefail
# Build this repository inside Linux and Illumos VMs using the Solstice dev loop.
# - Cross-build runner for both targets
# - Serve both runner binaries locally
# - Start orchestrator and enqueue two jobs (ubuntu-22.04, illumos-latest)
# - Each VM will download the appropriate runner and execute .solstice/job.sh from this repo

ROOT_DIR=$(cd "$(dirname "$0")/../../.." && pwd)
cd "$ROOT_DIR"

# Requirements check
command -v cargo >/dev/null 2>&1 || { echo "cargo is required" >&2; exit 127; }
command -v cross >/dev/null 2>&1 || { echo "cross is required (cargo install cross)" >&2; exit 127; }

# Defaults
export RUST_LOG=${RUST_LOG:-info}
export ORCH_CONFIG=${ORCH_CONFIG:-examples/orchestrator-image-map.yaml}
export AMQP_URL=${AMQP_URL:-amqp://127.0.0.1:5672/%2f}
export AMQP_EXCHANGE=${AMQP_EXCHANGE:-solstice.jobs}
export AMQP_QUEUE=${AMQP_QUEUE:-solstice.jobs.v1}
export AMQP_ROUTING_KEY=${AMQP_ROUTING_KEY:-jobrequest.v1}
export AMQP_PREFETCH=${AMQP_PREFETCH:-2}
export GRPC_ADDR=${GRPC_ADDR:-0.0.0.0:50051}

# Detect host IP for guest access (virbr0 first)
if command -v ip >/dev/null 2>&1 && ip addr show virbr0 >/dev/null 2>&1; then
  HOST_IP=$(ip -o -4 addr show virbr0 | awk '{print $4}' | cut -d/ -f1 | head -n1)
else
  HOST_IP=${HOST_IP_OVERRIDE:-127.0.0.1}
fi

# Orchestrator contact address for gRPC log streaming from guests
export ORCH_CONTACT_ADDR=${ORCH_CONTACT_ADDR:-$HOST_IP:50051}

# Bring up RabbitMQ
"$ROOT_DIR/.mise/tasks/dev/up"

# Ensure cleanup
ORCH_PID=""
SERVE_PID=""
cleanup() {
  set +e
  if [[ -n "$ORCH_PID" ]] && kill -0 "$ORCH_PID" 2>/dev/null; then
    echo "Stopping orchestrator (pid=$ORCH_PID)" >&2
    kill "$ORCH_PID" 2>/dev/null || true
    sleep 1
    kill -9 "$ORCH_PID" 2>/dev/null || true
  fi
  if [[ -n "$SERVE_PID" ]] && kill -0 "$SERVE_PID" 2>/dev/null; then
    echo "Stopping runner servers (pid=$SERVE_PID)" >&2
    kill "$SERVE_PID" 2>/dev/null || true
  fi
  "$ROOT_DIR/.mise/tasks/dev/down" || true
}
trap cleanup EXIT INT TERM

# Cross-build runner for both targets
"$ROOT_DIR/.mise/tasks/build/runner-cross"

# Start multi runner servers (background)
SOL_RUNNER_PORT_LINUX=${SOL_RUNNER_PORT_LINUX:-8090}
SOL_RUNNER_PORT_ILLUMOS=${SOL_RUNNER_PORT_ILLUMOS:-8091}
(
  exec "$ROOT_DIR/.mise/tasks/run/runner-serve-multi" >/dev/null 2>&1
) &
SERVE_PID=$!

# Compose URLs for both OSes and export for orchestrator cloud-init consumption
LINUX_URL="http://$HOST_IP:$SOL_RUNNER_PORT_LINUX/solstice-runner-linux"
ILLUMOS_URL="http://$HOST_IP:$SOL_RUNNER_PORT_ILLUMOS/solstice-runner-illumos"
export SOLSTICE_RUNNER_URLS="$LINUX_URL $ILLUMOS_URL"

# Start orchestrator in background (inherits env including SOLSTICE_RUNNER_URLS/ORCH_CONTACT_ADDR)
# Speed up startup by skipping persistence unless explicitly disabled
export ORCH_SKIP_PERSIST=${ORCH_SKIP_PERSIST:-true}
LOGFILE=${SOL_ORCH_LOG:-"$ROOT_DIR/target/orchestrator.vm-build.log"}
echo "Starting orchestrator... (logs: $LOGFILE)" >&2
(
  exec "$ROOT_DIR/.mise/tasks/run/orchestrator" >"$LOGFILE" 2>&1
) &
ORCH_PID=$!

echo "Runner URLs:" >&2
echo "  Linux:   $LINUX_URL" >&2
echo "  Illumos: $ILLUMOS_URL" >&2
echo "Orchestrator contact: $ORCH_CONTACT_ADDR" >&2

echo "Waiting for orchestrator to start..." >&2
# shellcheck disable=SC2034
for i in {1..20}; do
  if grep -q "orchestrator starting" "$LOGFILE" 2>/dev/null; then break; fi
  sleep 0.5
done

# Enqueue two jobs: one Linux, one Illumos and capture their request IDs
SOL_REPO_URL=${SOL_REPO_URL:-$(git -C "$ROOT_DIR" remote get-url origin 2>/dev/null || true)}
SOL_COMMIT_SHA=${SOL_COMMIT_SHA:-$(git -C "$ROOT_DIR" rev-parse HEAD 2>/dev/null || true)}
if [[ -z "${SOL_REPO_URL}" || -z "${SOL_COMMIT_SHA}" ]]; then
  echo "Warning: could not detect repo URL/commit; forge-enqueue will attempt autodetect" >&2
fi

# Linux (Ubuntu image in example config)
REQ_LINUX=$(SOL_RUNS_ON=ubuntu-22.04 "$ROOT_DIR/.mise/tasks/run/forge-enqueue")
# Illumos (default label / alias)
REQ_ILLUMOS=$(SOL_RUNS_ON=illumos-latest "$ROOT_DIR/.mise/tasks/run/forge-enqueue")

echo "Enqueued request IDs:" >&2
echo "  Linux:   $REQ_LINUX" >&2
echo "  Illumos: $REQ_ILLUMOS" >&2

# Wait for both jobs to finish by watching orchestrator logs
TIMEOUT_SECS=${SOL_WAIT_TIMEOUT_SECS:-3600}
DEADLINE=$(( $(date +%s) + TIMEOUT_SECS ))
DONE_LINUX=""
DONE_ILLUMOS=""
STATUS_LINUX=""
STATUS_ILLUMOS=""

echo "Waiting up to ${TIMEOUT_SECS}s for jobs to finish..." >&2
while :; do
  now=$(date +%s)
  if [[ $now -ge $DEADLINE ]]; then
    echo "Timeout waiting for jobs to finish" >&2
    exit 124
  fi
  if [[ -z "$DONE_LINUX" ]]; then
    if grep -q "job finished: ${REQ_LINUX} succeeded" "$LOGFILE"; then DONE_LINUX=1; STATUS_LINUX=ok; fi
    if grep -q "job finished: ${REQ_LINUX} failed" "$LOGFILE"; then DONE_LINUX=1; STATUS_LINUX=failed; fi
  fi
  if [[ -z "$DONE_ILLUMOS" ]]; then
    if grep -q "job finished: ${REQ_ILLUMOS} succeeded" "$LOGFILE"; then DONE_ILLUMOS=1; STATUS_ILLUMOS=ok; fi
    if grep -q "job finished: ${REQ_ILLUMOS} failed" "$LOGFILE"; then DONE_ILLUMOS=1; STATUS_ILLUMOS=failed; fi
  fi
  if [[ -n "$DONE_LINUX" && -n "$DONE_ILLUMOS" ]]; then
    break
  fi
  sleep 2
done

echo "Job results:" >&2
echo "  Linux:   $STATUS_LINUX" >&2
echo "  Illumos: $STATUS_ILLUMOS" >&2

# Exit non-zero if any failed
if [[ "$STATUS_LINUX" != "ok" || "$STATUS_ILLUMOS" != "ok" ]]; then
  echo "One or more jobs failed" >&2
  exit 1
fi

echo "Done. Logs at $LOGFILE" >&2