Map per-step log ranges to YAML steps using KDL step order

- Streamer sorts step categories in KDL workflow order (not alphabetical)
- Reporter emits one StepState per KDL step, each mapped by position
  to the corresponding YAML step ID
- Setup logs auto-map to "Set up job", per-step logs to their steps
This commit is contained in:
Till Wegmueller 2026-04-07 00:41:26 +02:00
parent ea3a249918
commit 49c3ab03c4
3 changed files with 76 additions and 38 deletions

View file

@ -120,12 +120,14 @@ pub async fn run(
let s_state = state.clone();
let s_request_id = jr.request_id;
let s_logs_base = logs_base.clone();
let s_steps = steps.clone();
tokio::spawn(crate::streamer::stream_logs(
s_client,
s_state,
s_request_id,
task_id,
s_logs_base,
s_steps,
stream_stop_rx,
));
}

View file

@ -200,28 +200,42 @@ async fn report_to_forgejo(
_ => vec![],
};
// Count lines per phase to set step boundaries
// Count setup lines (goes to Forgejo's virtual "Set up job")
let setup_lines: i64 = categories
.iter()
.filter(|c| SETUP_CATEGORIES.contains(&c.category.as_str()))
.map(|c| c.count)
.sum();
// Build per-step boundaries matching KDL step order.
// Each KDL step maps to a YAML step by position (id=0,1,2...).
// The streamer sorts categories in the same order, so indices align.
let mut cursor = setup_lines;
for (step_idx, step_info) in task_meta.steps.iter().enumerate() {
let step_lines = categories
.iter()
.find(|c| c.category == step_info.log_category)
.map(|c| c.count)
.unwrap_or(0);
step_states.push(StepState {
id: step_idx as i64,
result: result as i32,
started_at: Some(now.clone()),
stopped_at: Some(now.clone()),
log_index: cursor,
log_length: step_lines,
});
cursor += step_lines;
}
// If no steps were tracked (e.g. tier 2), use a single step for all work
if task_meta.steps.is_empty() {
let work_lines: i64 = categories
.iter()
.filter(|c| !SETUP_CATEGORIES.contains(&c.category.as_str()))
.map(|c| c.count)
.sum();
total_lines = setup_lines + work_lines;
// Forgejo's "Set up job" and "Complete job" are virtual UI steps —
// they auto-collect logs outside any real step's range. Only actual
// YAML steps need StepState entries with matching IDs.
//
// The YAML trigger has 1 step (id=0). Logs before its log_index
// go to "Set up job", logs in its range go to the step, and
// any logs after go to "Complete job".
step_states.push(StepState {
id: 0,
result: result as i32,
@ -230,6 +244,21 @@ async fn report_to_forgejo(
log_index: setup_lines,
log_length: work_lines,
});
cursor = setup_lines + work_lines;
}
// Account for any remaining categories not mapped to steps
let remaining: i64 = categories
.iter()
.filter(|c| {
!SETUP_CATEGORIES.contains(&c.category.as_str())
&& !task_meta.steps.iter().any(|s| s.log_category == c.category)
})
.map(|c| c.count)
.sum();
cursor += remaining;
total_lines = cursor;
}
// Send final "no more logs" marker

View file

@ -6,7 +6,7 @@ use uuid::Uuid;
use crate::connect::ConnectClient;
use crate::proto::runner::v1::{LogRow, UpdateLogRequest};
use crate::state::RunnerState;
use crate::state::{RunnerState, StepInfo};
const POLL_INTERVAL: Duration = Duration::from_secs(3);
@ -22,16 +22,15 @@ struct LogCategorySummary {
/// Streams logs from logs-service to Forgejo while a job is in-flight.
///
/// On each poll, fetches ALL log lines (sorted: setup categories first, then
/// work categories) and only sends lines beyond what Forgejo already has.
/// This ensures log indices always align with the reporter's step boundaries
/// regardless of the order categories appear in the DB.
/// Categories are ordered: setup categories first, then step categories in
/// KDL workflow order (matching the YAML step order), then any remaining.
pub async fn stream_logs(
client: Arc<ConnectClient>,
state: Arc<RunnerState>,
request_id: Uuid,
task_id: i64,
logs_base: String,
steps: Vec<StepInfo>,
mut stop: tokio::sync::watch::Receiver<bool>,
) -> i64 {
let http = reqwest::Client::new();
@ -40,14 +39,14 @@ pub async fn stream_logs(
loop {
if *stop.borrow() {
log_index = poll_and_send(
&client, &state, &http, &logs_base, request_id, task_id, log_index,
&client, &state, &http, &logs_base, request_id, task_id, log_index, &steps,
)
.await;
break;
}
log_index = poll_and_send(
&client, &state, &http, &logs_base, request_id, task_id, log_index,
&client, &state, &http, &logs_base, request_id, task_id, log_index, &steps,
)
.await;
@ -56,7 +55,7 @@ pub async fn stream_logs(
_ = stop.changed() => {
log_index = poll_and_send(
&client, &state, &http, &logs_base,
request_id, task_id, log_index,
request_id, task_id, log_index, &steps,
).await;
break;
}
@ -67,6 +66,22 @@ pub async fn stream_logs(
log_index
}
/// Sort categories: setup first, then step categories in KDL order, then any remaining.
fn sort_categories(categories: &mut [LogCategorySummary], steps: &[StepInfo]) {
categories.sort_by_key(|c| {
if SETUP_CATEGORIES.contains(&c.category.as_str()) {
// Setup categories come first, sub-sorted alphabetically
(0, 0, c.category.clone())
} else if let Some(pos) = steps.iter().position(|s| s.log_category == c.category) {
// Step categories in KDL workflow order
(1, pos, c.category.clone())
} else {
// Any remaining categories at the end
(2, 0, c.category.clone())
}
});
}
async fn poll_and_send(
client: &ConnectClient,
state: &RunnerState,
@ -75,6 +90,7 @@ async fn poll_and_send(
request_id: Uuid,
task_id: i64,
current_index: i64,
steps: &[StepInfo],
) -> i64 {
let categories_url = format!(
"{}/jobs/{}/logs",
@ -94,15 +110,7 @@ async fn poll_and_send(
return current_index;
}
// Sort: setup categories first, then work categories.
// This order must match the reporter's step boundary calculation.
categories.sort_by_key(|c| {
if SETUP_CATEGORIES.contains(&c.category.as_str()) {
(0, c.category.clone())
} else {
(1, c.category.clone())
}
});
sort_categories(&mut categories, steps);
// Build the full ordered log by fetching each category
let mut all_lines: Vec<String> = Vec::new();
@ -132,7 +140,6 @@ async fn poll_and_send(
return current_index;
}
// Only send lines beyond what we've already sent
let new_lines = &all_lines[current_index as usize..];
let now = prost_types::Timestamp {