From c0bac67bd93294390826f975647024f19d9e7572 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola <144015745+tomastola@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:08:54 +0200 Subject: [PATCH 1/4] ci(xtask): add e2e-queue-import to turn a queue run into nightly report inputs Reads a finished run of the shared GPU queue (its per-lane results artifact and the run's job log) and writes each lane's report files where `cargo xtask e2e-report` reads them, named for the lane and the queue's gate, plus a status section for the top of the support matrix. Not wired in yet. Signed-off-by: Tomas Saaristola --- xtask/src/e2e_queue_import.rs | 931 ++++++++++++++++++++++++++++++++++ 1 file changed, 931 insertions(+) create mode 100644 xtask/src/e2e_queue_import.rs diff --git a/xtask/src/e2e_queue_import.rs b/xtask/src/e2e_queue_import.rs new file mode 100644 index 000000000..a5f0bc459 --- /dev/null +++ b/xtask/src/e2e_queue_import.rs @@ -0,0 +1,931 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Bring a finished run of the shared GPU queue into the nightly's consolidated +//! E2E report, so the support matrix and the release-readiness verdict derived +//! from it keep covering the GPUs the queue tests. +//! +//! `nightly.yml`'s `e2e-report-nightly` merges one `report.json` per +//! `e2e-*-report` artifact. The queue publishes a different layout: one +//! directory per lane in its `queue-run-results` artifact, holding +//! `/results/{report.json,platform.json,commands.jsonl,...}` as +//! `scripts/run-e2e-lane.sh` writes them (docs/nightly-on-the-queue.md). This +//! turns the one into the other: +//! +//! * every lane's reports are copied to `/e2e-queue-[-advisory]--report/`, +//! named so `e2e_report` labels them by lane and gate. Only the three files the +//! report reads are copied; `lane.log`, the JUnit and HTML output and the +//! harness diagnostics stay where they are. +//! * the report's `platform_slug` is replaced by the lane's name and channel. The +//! slug is what the report keys a grid column by, and queue lanes of different +//! models report one slug (an MI325X lane probes as `mi300x`, an MI355X as +//! `mi350p`), which would fold them, and the self-hosted MI300X lane, into one +//! column. The channel is there because a lane's manifest names the channel of +//! the runtime it found, which need not be the one the lane was told to run. +//! * which lanes exist and which of them the queue blocks on is read from the +//! run's lane matrix in its job log, the only place either is published. A lane +//! the queue marks advisory is labelled so, and `release-readiness` leaves it +//! out of its verdict. Anything this cannot read counts as blocking. +//! * a lane that reported nothing, or that did not finish cleanly and lacks a +//! channel, gets an empty report for it, which the report shows as `EMPTY` and +//! `release-readiness` counts as a failure for a blocking lane: a lane without +//! a result is not a pass. +//! * a short status table of every lane is written for the top of the page. +//! +//! Nothing is imported unless the caller found a run: with `--absent ` +//! only the status section is written, saying so, and the nightly report goes on +//! without queue lanes. The repository is public, so everything written is either +//! copied from the queue's own scanned files or built from fields this checks +//! against a strict pattern first. + +use std::collections::BTreeMap; +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result, bail}; +use regex::Regex; + +/// The channels `scripts/run-e2e-lane.sh` runs, in the order it runs them. +const CHANNELS: [&str; 2] = ["release", "nightly"]; + +/// The files of a lane's `/results` the consolidated report reads. +const REPORT_FILES: [&str; 3] = ["report.json", "platform.json", "commands.jsonl"]; + +#[derive(clap::Args)] +pub struct Args { + /// Directory the queue run's `queue-run-results` artifact was downloaded + /// into: one subdirectory per lane. + #[arg(long, requires = "run_url", conflicts_with = "absent")] + queue_dir: Option, + /// The queue run's job log, which prints the branch the run tested and its + /// lane matrix. Without it, or when it names a branch other than `main`, + /// nothing is imported. + #[arg(long, requires = "queue_dir")] + log: Option, + /// The queue run's URL, linked from the status section. + #[arg(long)] + run_url: Option, + /// When the queue run started, as the API reports it. + #[arg(long, requires = "queue_dir")] + run_started: Option, + /// Import nothing and say why: no queue run was found, or its results could + /// not be fetched. + #[arg( + long, + conflicts_with = "queue_dir", + required_unless_present = "queue_dir" + )] + absent: Option, + /// Where the per-lane report directories go: the directory + /// `xtask e2e-report` reads. + #[arg(long)] + artifacts_dir: PathBuf, + /// Where to write the status section, for the top of the support matrix. + #[arg(long)] + section_out: PathBuf, +} + +pub fn run(args: &Args) -> Result<()> { + let section = match (&args.queue_dir, &args.absent) { + (Some(queue_dir), _) => { + let log = args.log.as_ref().and_then(|path| { + std::fs::read_to_string(path) + .map_err(|e| eprintln!("warning: queue job log {}: {e}", path.display())) + .ok() + }); + let log = parse_log(log.as_deref()); + if log.branch.as_deref() == Some("main") { + // A log that yielded no lanes is as good as none for the lanes' + // state: every lane is then read as blocking. + let have_matrix = !log.lanes.is_empty(); + let plan = plan(&log, scan(queue_dir)?, have_matrix); + write_reports(&plan, queue_dir, &args.artifacts_dir)?; + render_section( + &plan, + args.run_url.as_deref(), + args.run_started.as_deref(), + have_matrix, + ) + } else { + // The run is of the workflow on `main`, but may have been asked + // to test another branch, which is no night's result for `main`. + // The log is the only place that says which, so without it + // nothing is imported. + render_absent( + "the newest queue run did not test main, or its job log could not be read \ + to tell", + ) + } + } + (None, Some(reason)) => render_absent(reason), + (None, None) => bail!("one of --queue-dir and --absent is required"), + }; + if let Some(parent) = args.section_out.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating {}", parent.display()))?; + } + std::fs::write(&args.section_out, section) + .with_context(|| format!("writing {}", args.section_out.display()))?; + Ok(()) +} + +/// What the queue run's job log says about the run. +#[derive(Debug, Default, PartialEq, Eq)] +struct QueueLog { + /// The branch the run tested and the commit every lane tested, from the + /// report's first line. The branch is not the one the workflow ran on: a run + /// started from `main` can be asked to test another branch. + branch: Option, + commit: Option, + lanes: Vec, +} + +/// One row of the lane matrix the queue prints. +#[derive(Debug, PartialEq, Eq)] +struct MatrixRow { + lane: String, + /// The lane's phase (`Done`, `Failed`, ...), when the row had the usual shape. + phase: Option, + /// The lane command's exit status, or `-` when it never ran. + exit: Option, + /// Whether the queue lets this lane fail the run. Anything but `advisory` is + /// read as blocking. + blocking: bool, +} + +impl MatrixRow { + /// A lane the queue reports as finished with exit status zero. + fn is_clean(&self) -> bool { + self.phase.as_deref() == Some("Done") && self.exit.as_deref() == Some("0") + } +} + +/// A lane name is a path segment and a word in an artifact name: lowercase +/// letters, digits and dashes only. +/// +/// A name ending in a word the artifact names use for something else +/// (`-advisory`, `-release`, `-nightly`) is refused too: it would be read back +/// as a gate or a channel. +fn is_lane_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= 40 + && !name.starts_with('-') + && !["-advisory", "-release", "-nightly"] + .iter() + .any(|suffix| name.ends_with(suffix)) + && name + .bytes() + .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') +} + +/// Read the lane matrix and the tested commit out of the run's job log. +/// +/// The publisher prints its report with a `| ` prefix on every line, which the +/// runner puts a timestamp in front of. Only the fields that match a strict +/// pattern are kept; a line that does not is ignored, so a log that is not what +/// was expected yields no lanes rather than text to publish. +fn parse_log(log: Option<&str>) -> QueueLog { + let mut out = QueueLog::default(); + let Some(log) = log else { return out }; + let tested = Regex::new(r"^rocm-cli (\S+) at ([0-9a-f]{40})$").expect("static pattern"); + let mut in_matrix = false; + let mut unnamed = 0; + for line in log.lines() { + let line = line.trim_start_matches('\u{feff}'); + // ` | `: the part after the first `| `. + let Some((_, body)) = line.split_once("| ") else { + // A blank report line is `|` alone; either way the matrix is over. + if line.trim_end().ends_with('|') { + in_matrix = false; + } + continue; + }; + let body = body.trim_end(); + if let Some(caps) = tested.captures(body) { + out.branch = Some(caps[1].to_owned()); + out.commit = Some(caps[2].to_owned()); + } + let tokens: Vec<&str> = body.split_whitespace().collect(); + if tokens.first() == Some(&"LANE") && tokens.last() == Some(&"GATE") { + in_matrix = true; + continue; + } + if !in_matrix { + continue; + } + if body.is_empty() || tokens.first() == Some(&"SUMMARY:") { + in_matrix = false; + continue; + } + let (Some(lane), Some(gate)) = (tokens.first(), tokens.last()) else { + continue; + }; + if out.lanes.iter().any(|row| row.lane == *lane) { + continue; + } + // A row whose lane cannot be named safely still counts: as a lane that + // reported nothing, and blocking, since its gate cannot be trusted either. + if !is_lane_name(lane) { + unnamed += 1; + out.lanes.push(MatrixRow { + lane: format!("unnamed-{unnamed}"), + phase: None, + exit: None, + blocking: true, + }); + continue; + } + // LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE + let shaped = tokens.len() == 7; + let phase = shaped + .then(|| tokens[2]) + .filter(|p| p.bytes().all(|b| b.is_ascii_alphabetic())) + .map(str::to_owned); + let exit = shaped + .then(|| tokens[3]) + .filter(|e| *e == "-" || (e.len() <= 4 && e.bytes().all(|b| b.is_ascii_digit()))) + .map(str::to_owned); + out.lanes.push(MatrixRow { + lane: (*lane).to_owned(), + phase, + exit, + blocking: *gate != "advisory", + }); + } + out +} + +/// The channels with a `results/report.json` under each lane directory of the +/// downloaded artifact. A directory (or the final file) that is a symlink is not +/// followed. A lane directory with a name that is not a lane name is kept as an +/// `unnamed-N` lane with no reports, so that it shows as a lane without a result +/// instead of dropping out unseen. +fn scan(queue_dir: &Path) -> Result>> { + let mut lanes = BTreeMap::new(); + let mut unnamed = 0; + let entries = std::fs::read_dir(queue_dir) + .with_context(|| format!("reading the queue results in {}", queue_dir.display()))?; + for entry in entries { + let entry = entry.context("reading a queue results entry")?; + if !entry.file_type().is_ok_and(|t| t.is_dir()) { + continue; + } + let Some(lane) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + if !is_lane_name(&lane) { + eprintln!("warning: a queue lane directory has an unexpected name; shown as unnamed"); + unnamed += 1; + lanes.insert(format!("unnamed-{unnamed}"), Vec::new()); + continue; + } + let channels = CHANNELS + .into_iter() + .filter(|channel| regular_file(&entry.path().join(channel).join("results/report.json"))) + .collect(); + lanes.insert(lane, channels); + } + Ok(lanes) +} + +fn regular_file(path: &Path) -> bool { + std::fs::symlink_metadata(path).is_ok_and(|m| m.is_file()) +} + +/// What one lane contributes to the report. +#[derive(Debug, PartialEq, Eq)] +struct LanePlan { + lane: String, + blocking: bool, + phase: Option, + exit: Option, + /// Channels whose reports are copied. + reported: Vec<&'static str>, + /// Channels (or, for `None`, the lane as a whole) an empty report stands in for. + placeholders: Vec>, +} + +#[derive(Debug, PartialEq, Eq)] +struct Plan { + commit: Option, + lanes: Vec, +} + +/// Decide, lane by lane, what goes into the report. `have_matrix` says whether the +/// lane matrix could be read at all: without it nothing is known of a lane's +/// state beyond its files. +fn plan(log: &QueueLog, found: BTreeMap>, have_matrix: bool) -> Plan { + let mut names: Vec<&String> = found + .keys() + .chain(log.lanes.iter().map(|r| &r.lane)) + .collect(); + names.sort(); + names.dedup(); + let mut lanes = Vec::new(); + for name in names { + let row = log.lanes.iter().find(|r| &r.lane == name); + let reported = found.get(name).cloned().unwrap_or_default(); + let clean = row.is_some_and(MatrixRow::is_clean); + let mut placeholders = Vec::new(); + if reported.is_empty() { + // Nothing came back from this lane, whatever it was told to run. + placeholders.push(None); + } else if have_matrix && !clean { + // It did not finish cleanly, and a channel it should have run left + // no report. A lane that finished cleanly may have been told to run + // one channel only. + placeholders.extend( + CHANNELS + .into_iter() + .filter(|channel| !reported.contains(channel)) + .map(Some), + ); + } + lanes.push(LanePlan { + lane: name.clone(), + blocking: row.is_none_or(|r| r.blocking), + phase: row.and_then(|r| r.phase.clone()), + exit: row.and_then(|r| r.exit.clone()), + reported, + placeholders, + }); + } + Plan { + commit: log.commit.clone(), + lanes, + } +} + +/// Write each lane's report directories under `artifacts_dir`. +fn write_reports(plan: &Plan, queue_dir: &Path, artifacts_dir: &Path) -> Result<()> { + for lane in &plan.lanes { + for channel in &lane.reported { + let dest = artifacts_dir.join(e2e_report::queue_artifact_name( + &lane.lane, + !lane.blocking, + Some(channel), + )); + std::fs::create_dir_all(&dest) + .with_context(|| format!("creating {}", dest.display()))?; + let src = queue_dir.join(&lane.lane).join(channel).join("results"); + for file in REPORT_FILES { + let from = src.join(file); + if !regular_file(&from) { + continue; + } + if file == "platform.json" { + // A manifest that does not parse is left out: the report then + // names the lane as having no usable manifest. + let text = std::fs::read_to_string(&from) + .with_context(|| format!("reading {}", from.display()))?; + if let Some(text) = with_lane_slug(&text, &lane.lane, channel) { + std::fs::write(dest.join(file), text) + .with_context(|| format!("writing {}", dest.join(file).display()))?; + } + } else { + std::fs::copy(&from, dest.join(file)) + .with_context(|| format!("copying {}", from.display()))?; + } + } + } + for channel in &lane.placeholders { + let dest = artifacts_dir.join(e2e_report::queue_artifact_name( + &lane.lane, + !lane.blocking, + *channel, + )); + std::fs::create_dir_all(&dest) + .with_context(|| format!("creating {}", dest.display()))?; + // No scenarios at all: the report shows the lane as EMPTY. + std::fs::write(dest.join("report.json"), "[]\n") + .with_context(|| format!("writing {}", dest.join("report.json").display()))?; + } + } + Ok(()) +} + +/// `platform.json` with its `platform_slug` (and the copy under `capability`) +/// set to the lane's name and channel, or `None` when it is not a JSON object. +fn with_lane_slug(text: &str, lane: &str, channel: &str) -> Option { + let mut manifest: serde_json::Value = serde_json::from_str(text).ok()?; + let slug = serde_json::Value::String(format!("queue-{lane}-{channel}")); + let object = manifest.as_object_mut()?; + object.insert("platform_slug".to_owned(), slug.clone()); + if let Some(capability) = object + .get_mut("capability") + .and_then(serde_json::Value::as_object_mut) + { + capability.insert("platform_slug".to_owned(), slug); + } + serde_json::to_string_pretty(&manifest).ok() +} + +/// Text that fits in a table cell, from a reason this repository wrote itself. +fn one_line(text: &str) -> String { + text.chars() + .map(|c| { + if c.is_ascii_graphic() || c == ' ' { + c + } else { + ' ' + } + }) + .filter(|c| *c != '|' && *c != '`') + .take(200) + .collect::() + .trim() + .to_owned() +} + +const SECTION_HEADING: &str = "## Queue lanes\n\n"; + +/// The section for a nightly with no queue results to import. +fn render_absent(reason: &str) -> String { + format!( + "{SECTION_HEADING}_No queue lanes are in this report: {}. The GPUs the shared queue \ + tests are not shown tonight._\n\n", + one_line(reason) + ) +} + +fn is_run_url(url: &str) -> bool { + Regex::new(r"^https://github\.com/[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+/actions/runs/[0-9]+$") + .expect("static pattern") + .is_match(url) +} + +/// The status section: which run was imported, and one row per lane. +fn render_section( + plan: &Plan, + run_url: Option<&str>, + run_started: Option<&str>, + have_matrix: bool, +) -> String { + let mut out = String::from(SECTION_HEADING); + let mut source = String::from("Latest finished run of the shared GPU queue on `main`"); + if let Some(url) = run_url.filter(|u| is_run_url(u)) { + let id = url.rsplit('/').next().unwrap_or_default(); + let _ = write!(source, ": [run {id}]({url})"); + } + if let Some(started) = run_started.filter(|s| { + Regex::new(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]{8}Z$") + .expect("static pattern") + .is_match(s) + }) { + let _ = write!(source, ", started {started}"); + } + if let Some(commit) = &plan.commit { + let _ = write!(source, ", tested commit `{}`", &commit[..12]); + } + let _ = writeln!(out, "{source}.\n"); + if !have_matrix { + out.push_str( + "_The run's lane matrix could not be read, so every lane is shown as blocking._\n\n", + ); + } + out.push_str("| Lane | GPU | Gate | Phase | Exit | Channels reported |\n"); + out.push_str("|---|---|---|---|---|---|\n"); + for lane in &plan.lanes { + let channels = if lane.reported.is_empty() { + "none".to_owned() + } else { + lane.reported.join(", ") + }; + let _ = writeln!( + out, + "| {} | {} | {} | {} | {} | {channels} |", + lane.lane, + e2e_report::queue_lane_gpu(&lane.lane), + if lane.blocking { + "blocking" + } else { + "advisory" + }, + lane.phase.as_deref().unwrap_or("unknown"), + lane.exit.as_deref().unwrap_or("unknown"), + ); + } + let _ = write!( + out, + "\nThe queue's own gate decides which lanes count: a lane it marks advisory is labelled \ + `{}` below and is left out of the release-readiness verdict, and a blocking lane that \ + reported no result is shown as `EMPTY` and counts as a failure there.\n\n", + e2e_report::ADVISORY_QUEUE_MARKER + ); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The lane matrix of the 2026-11-13 scheduled run, as its public job log + /// printed it. + const LOG: &str = "\u{feff}2026-11-13T02:22:14.0268419Z started gh-43351651832-1; waiting for it to finish +2026-11-13T03:45:26.1317642Z gh-43351651832-1 still running (10 min, 0 of 7 lane(s) finished) +2026-11-13T03:45:26.1323838Z request gh-43351651832-1, finished 2026-11-13T03:45:26Z +2026-11-13T03:45:26.1361153Z | image: \"rocm/dev-ubuntu-24.04:7.2.4\" +2026-11-13T03:45:26.1361751Z | blocking: \"mi300x,r9700s\" +2026-11-13T03:45:26.1362299Z | rocm-cli main at 691834a04540743d0dced173bd2ef70c1244b351 +2026-11-13T03:45:26.1362503Z | lanes: mi300x mi325x mi350p mi350x mi355x r9700s w7900-dual +2026-11-13T03:45:26.1362739Z | +2026-11-13T03:45:26.1362910Z | LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE +2026-11-13T03:45:26.1363201Z | mi300x rocm-cli-gh-43351651832-1-mi300x Failed 1 Exported 0m block +2026-11-13T03:45:26.1363511Z | mi325x rocm-cli-gh-43351651832-1-mi325x Failed 1 Exported 0m advisory +2026-11-13T03:45:26.1364685Z | r9700s rocm-cli-gh-43351651832-1-r9700s Failed - - 0m block +2026-11-13T03:45:26.1364991Z | w7900-dual rocm-cli-gh-43351651832-1-w7900-dual Done 0 Exported 0m advisory +2026-11-13T03:45:26.1365442Z | +2026-11-13T03:45:26.1365634Z | SUMMARY: 7 lane(s), 1 passed, 2 failed, 0 incomplete, 0 missing, 4 advisory +2026-11-13T03:45:26.1365895Z | lane mi300x: 7 file(s) collected +2026-11-13T03:45:26.1365895Z | mi999x not a row after the matrix Failed 1 Exported 0m block +"; + + fn row(lane: &str, phase: &str, exit: &str, blocking: bool) -> MatrixRow { + MatrixRow { + lane: lane.to_owned(), + phase: Some(phase.to_owned()), + exit: Some(exit.to_owned()), + blocking, + } + } + + #[test] + fn the_matrix_and_the_tested_commit_are_read_from_the_job_log() { + let log = parse_log(Some(LOG)); + assert_eq!(log.branch.as_deref(), Some("main")); + assert_eq!( + log.commit.as_deref(), + Some("691834a04540743d0dced173bd2ef70c1244b351") + ); + assert_eq!( + log.lanes, + vec![ + row("mi300x", "Failed", "1", true), + row("mi325x", "Failed", "1", false), + row("r9700s", "Failed", "-", true), + row("w7900-dual", "Done", "0", false), + ], + "a line after the matrix is not a lane" + ); + } + + #[test] + fn a_log_that_is_not_the_report_yields_nothing_to_publish() { + assert_eq!(parse_log(None), QueueLog::default()); + assert_eq!( + parse_log(Some( + "ERR_ACCESS_DENIED\nLANE GATE\nmi300x x\n" + )), + QueueLog::default() + ); + // A row of an unexpected shape keeps its lane and its gate, and nothing else. + let odd = "2026-11-13T03:45:26Z | LANE WORKLOAD PHASE GATE\n\ + 2026-11-13T03:45:26Z | mi300x some workload Pending now advisory\n"; + assert_eq!( + parse_log(Some(odd)).lanes, + vec![MatrixRow { + lane: "mi300x".to_owned(), + phase: None, + exit: None, + blocking: false, + }] + ); + // Text that is not a lane name is never kept, whatever the row says; the + // lane is counted, as blocking. + let hostile = "2026-11-13T03:45:26Z | LANE GATE\n\ + 2026-11-13T03:45:26Z | ../../etc/passwd advisory\n\ + 2026-11-13T03:45:26Z | Uppercase block\n"; + let lanes = parse_log(Some(hostile)).lanes; + assert_eq!( + lanes.iter().map(|l| l.lane.as_str()).collect::>(), + ["unnamed-1", "unnamed-2"] + ); + assert!(lanes.iter().all(|l| l.blocking)); + } + + fn found(entries: &[(&str, &[&'static str])]) -> BTreeMap> { + entries + .iter() + .map(|(lane, channels)| ((*lane).to_owned(), channels.to_vec())) + .collect() + } + + #[test] + fn a_blocking_lane_with_no_result_gets_an_empty_report_and_an_advisory_one_stays_advisory() { + let log = parse_log(Some(LOG)); + let plan = plan( + &log, + found(&[ + ("mi300x", &["release"]), + ("mi325x", &["release", "nightly"]), + ("w7900-dual", &["release", "nightly"]), + ]), + true, + ); + let by_lane = |name: &str| { + plan.lanes + .iter() + .find(|l| l.lane == name) + .expect("lane planned") + }; + // Failed, and the nightly channel left nothing: one stands in for it. + assert_eq!(by_lane("mi300x").placeholders, vec![Some("nightly")]); + assert!(by_lane("mi300x").blocking); + // Nothing at all came back from r9700s, which the queue never listed a file for. + assert_eq!(by_lane("r9700s").placeholders, vec![None]); + assert!(by_lane("r9700s").reported.is_empty()); + // Advisory stays advisory, and a lane with every channel needs no stand-in. + assert!(!by_lane("mi325x").blocking); + assert!(by_lane("mi325x").placeholders.is_empty()); + assert!(by_lane("w7900-dual").placeholders.is_empty()); + assert_eq!( + plan.commit.as_deref(), + Some("691834a04540743d0dced173bd2ef70c1244b351") + ); + } + + #[test] + fn a_lane_that_finished_cleanly_may_have_run_one_channel_only() { + let log = QueueLog { + branch: Some("main".to_owned()), + commit: None, + lanes: vec![row("mi300x", "Done", "0", true)], + }; + let plan = plan(&log, found(&[("mi300x", &["release"])]), true); + assert!(plan.lanes[0].placeholders.is_empty()); + } + + #[test] + fn without_the_matrix_every_lane_is_blocking_and_only_a_lane_with_no_files_is_missing() { + let plan = plan( + &QueueLog::default(), + found(&[("mi325x", &["release"]), ("r9700s", &[])]), + false, + ); + assert!(plan.lanes.iter().all(|l| l.blocking)); + assert!(plan.lanes[0].placeholders.is_empty()); + assert_eq!(plan.lanes[1].placeholders, vec![None]); + } + + #[test] + fn the_lane_and_channel_replace_the_slug_that_two_lanes_share() { + let manifest = r#"{"platform_slug":"mi300x","capability":{"platform_slug":"mi300x", + "gfx_target":"gfx942"},"versions":{"channel":"release"},"expectations":[]}"#; + let rewritten = with_lane_slug(manifest, "mi325x", "nightly").expect("an object"); + let value: serde_json::Value = serde_json::from_str(&rewritten).expect("json"); + assert_eq!(value["platform_slug"], "queue-mi325x-nightly"); + assert_eq!(value["capability"]["platform_slug"], "queue-mi325x-nightly"); + assert_eq!(value["capability"]["gfx_target"], "gfx942"); + assert_eq!(value["versions"]["channel"], "release"); + assert_eq!(with_lane_slug("[1]", "mi325x", "release"), None); + assert_eq!(with_lane_slug("not json", "mi325x", "release"), None); + } + + fn write(path: &Path, text: &str) { + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(path, text).expect("write"); + } + + /// A queue artifact as the run above published it, cut down to two lanes. + fn queue_artifact(dir: &Path) { + for (lane, slug, channels) in [ + ("mi300x", "mi300x", &["release"][..]), + ("mi325x", "mi300x", &["release", "nightly"][..]), + ] { + write(&dir.join(lane).join("lane.log"), "the lane's own log\n"); + for channel in channels { + let results = dir.join(lane).join(channel).join("results"); + write(&results.join("report.json"), "[]\n"); + write(&results.join("commands.jsonl"), ""); + write(&results.join("junit.xml"), ""); + write(&results.join("harness-diagnostics.log"), "diagnostics"); + write( + &results.join("platform.json"), + &format!( + r#"{{"platform_slug":"{slug}","versions":{{"channel":"{channel}"}},"expectations":[]}}"# + ), + ); + } + } + write(&dir.join("r9700s").join("lane.log"), "no files came back\n"); + } + + #[test] + fn the_import_writes_only_what_the_report_reads_under_names_that_say_lane_and_gate() { + let queue = tempfile::tempdir().expect("tempdir"); + queue_artifact(queue.path()); + let log = queue.path().join("job.log"); + // The log sits beside the lanes here; a stray file is no lane. + write(&log, LOG); + let out = tempfile::tempdir().expect("tempdir"); + let section = out.path().join("consolidated/queue-lanes.md"); + let args = Args { + queue_dir: Some(queue.path().to_owned()), + log: Some(log), + run_url: Some("https://github.com/ROCm/rocm-cli/actions/runs/43351651832".to_owned()), + run_started: Some("2026-11-13T02:21:58Z".to_owned()), + absent: None, + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }; + run(&args).expect("import"); + + let mut dirs: Vec = std::fs::read_dir(out.path().join("e2e-artifacts")) + .expect("artifacts") + .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + dirs.sort(); + assert_eq!( + dirs, + [ + "e2e-queue-mi300x-nightly-report", + "e2e-queue-mi300x-release-report", + "e2e-queue-mi325x-advisory-nightly-report", + "e2e-queue-mi325x-advisory-release-report", + "e2e-queue-r9700s-report", + // Listed in the log, with no files at all: an empty stand-in. + "e2e-queue-w7900-dual-advisory-report", + ] + ); + let copied = out + .path() + .join("e2e-artifacts/e2e-queue-mi325x-advisory-release-report"); + let mut files: Vec = std::fs::read_dir(&copied) + .expect("lane dir") + .map(|e| e.expect("entry").file_name().to_string_lossy().into_owned()) + .collect(); + files.sort(); + assert_eq!(files, ["commands.jsonl", "platform.json", "report.json"]); + let manifest = std::fs::read_to_string(copied.join("platform.json")).expect("manifest"); + assert!(manifest.contains("queue-mi325x-release"), "{manifest}"); + assert_eq!( + std::fs::read_to_string( + out.path() + .join("e2e-artifacts/e2e-queue-r9700s-report/report.json") + ) + .expect("placeholder"), + "[]\n" + ); + + let section = std::fs::read_to_string(section).expect("section"); + assert!(section.starts_with("## Queue lanes\n"), "{section}"); + assert!( + section.contains( + "[run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832), \ + started 2026-11-13T02:21:58Z, tested commit `691834a04540`" + ), + "{section}" + ); + assert!( + section.contains("| mi325x | MI325X | advisory | Failed | 1 | release, nightly |"), + "{section}" + ); + assert!( + section.contains("| r9700s | R9700 | blocking | Failed | - | none |"), + "{section}" + ); + // Nothing of the lane's own log or the workload names reaches the page. + assert!(!section.contains("rocm-cli-gh-"), "{section}"); + assert!(!section.contains("lane's own log"), "{section}"); + } + + #[test] + fn an_absent_queue_run_is_a_section_that_says_so_and_nothing_else() { + let out = tempfile::tempdir().expect("tempdir"); + let section = out.path().join("queue-lanes.md"); + run(&Args { + queue_dir: None, + log: None, + run_url: None, + run_started: None, + absent: Some("no queue run finished on main in the last 24 hours | `x`\n".to_owned()), + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }) + .expect("an absent run is not an error"); + assert!(!out.path().join("e2e-artifacts").exists()); + let text = std::fs::read_to_string(section).expect("section"); + assert_eq!( + text, + "## Queue lanes\n\n_No queue lanes are in this report: no queue run finished on \ + main in the last 24 hours x. The GPUs the shared queue tests are not shown \ + tonight._\n\n" + ); + } + + fn import_with_log(log_text: Option<&str>) -> (tempfile::TempDir, String) { + let queue = tempfile::tempdir().expect("tempdir"); + queue_artifact(queue.path()); + let out = tempfile::tempdir().expect("tempdir"); + let log = log_text.map(|text| { + let path = out.path().join("job.log"); + write(&path, text); + path + }); + let section = out.path().join("queue-lanes.md"); + run(&Args { + queue_dir: Some(queue.path().to_owned()), + log, + run_url: Some("https://github.com/ROCm/rocm-cli/actions/runs/1".to_owned()), + run_started: None, + absent: None, + artifacts_dir: out.path().join("e2e-artifacts"), + section_out: section.clone(), + }) + .expect("a log that cannot be used is not an error"); + let text = std::fs::read_to_string(section).expect("section"); + (out, text) + } + + #[test] + fn a_run_that_tested_another_branch_or_whose_log_is_unreadable_is_not_imported() { + // A run started from main can be asked to test any branch of the repository. + let other = LOG.replace("rocm-cli main at", "rocm-cli some-branch at"); + for log in [ + Some(other.as_str()), + // What the egress proxy answers when it refuses the request. + Some("ERR_ACCESS_DENIED\n"), + None, + ] { + let (out, text) = import_with_log(log); + assert!( + text.contains("did not test main, or its job log could not be read"), + "{text}" + ); + assert!(!text.contains("| Lane |"), "{text}"); + assert!(!out.path().join("e2e-artifacts").exists()); + } + } + + #[test] + fn a_log_without_a_matrix_leaves_every_lane_blocking_and_says_so() { + let (out, text) = import_with_log(Some( + "2026-11-13T03:45:26Z | rocm-cli main at 691834a04540743d0dced173bd2ef70c1244b351\n", + )); + assert!(text.contains("every lane is shown as blocking"), "{text}"); + assert!(!text.contains("advisory |"), "{text}"); + // The lane with no files is still a missing result; the others are not + // second-guessed about a channel they never ran. + let arts = out.path().join("e2e-artifacts"); + assert!(arts.join("e2e-queue-r9700s-report").is_dir()); + assert!(!arts.join("e2e-queue-mi300x-nightly-report").exists()); + } + + #[test] + fn a_lane_that_cannot_be_named_still_shows_as_a_blocking_lane_with_no_result() { + for name in [ + "Mi300X", + "mi300x_2", + "x-advisory", + "x-release", + "x-nightly", + "-x", + ] { + assert!(!is_lane_name(name), "{name}"); + } + let log = parse_log(Some( + "2026-11-13T03:45:26Z | LANE WORKLOAD PHASE EXIT EXPORT WAITED GATE\n\ + 2026-11-13T03:45:26Z | x-advisory w Done 0 Exported 0m advisory\n", + )); + assert_eq!( + log.lanes, + vec![MatrixRow { + lane: "unnamed-1".to_owned(), + phase: None, + exit: None, + blocking: true, + }] + ); + // The same for a directory of the artifact. + let queue = tempfile::tempdir().expect("tempdir"); + write(&queue.path().join("mi300x_2/lane.log"), "log\n"); + write( + &queue.path().join("mi300x/release/results/report.json"), + "[]", + ); + let found = scan(queue.path()).expect("scan"); + assert_eq!(found.get("unnamed-1"), Some(&Vec::new())); + assert_eq!(found.get("mi300x"), Some(&vec!["release"])); + let plan = plan(&QueueLog::default(), found, false); + let unnamed = plan + .lanes + .iter() + .find(|l| l.lane == "unnamed-1") + .expect("lane"); + assert!(unnamed.blocking); + assert_eq!(unnamed.placeholders, vec![None]); + } + + #[test] + fn a_run_url_that_is_not_a_run_is_not_linked() { + let plan = Plan { + commit: None, + lanes: Vec::new(), + }; + let text = render_section(&plan, Some("https://example.com/x)[y]("), None, true); + assert!(!text.contains("example.com"), "{text}"); + } +} From f8d83c10f6600c0dfd106291b74d53aeca58ff6e Mon Sep 17 00:00:00 2001 From: Tomas Saaristola <144015745+tomastola@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:11:23 +0200 Subject: [PATCH 2/4] feat(e2e-report): label queue lanes by lane and gate, and keep advisory ones out of the verdict The e2e report names a lane of the shared GPU queue by the lane (a queue MI300X lane is not the self-hosted one) and marks one the queue does not block on as advisory. release-readiness leaves advisory rows out of its failure count and still counts a blocking lane that came back with no result. Signed-off-by: Tomas Saaristola --- .../e2e-report/src/consolidated/descriptor.rs | 94 +++++++++++++++++++ crates/e2e-report/src/consolidated/mod.rs | 4 +- crates/e2e-report/src/lib.rs | 5 +- docs/architecture.md | 1 + xtask/src/main.rs | 8 ++ xtask/src/release_readiness.rs | 88 +++++++++++++---- 6 files changed, 181 insertions(+), 19 deletions(-) diff --git a/crates/e2e-report/src/consolidated/descriptor.rs b/crates/e2e-report/src/consolidated/descriptor.rs index 7291dd2cb..1185546de 100644 --- a/crates/e2e-report/src/consolidated/descriptor.rs +++ b/crates/e2e-report/src/consolidated/descriptor.rs @@ -48,6 +48,11 @@ pub(super) fn parse_descriptor(name: &str) -> Descriptor { }, }; + // A lane of the shared GPU queue (see `queue_descriptor`). + if let Some(lane) = core.strip_prefix("queue-") { + return queue_descriptor(lane, channel_from_name); + } + // Legacy `-known-bugs` suffix (retained only as a stable secondary sort key). let (core, known_bugs) = match core.strip_suffix("known-bugs") { Some(rest) => (rest.trim_end_matches('-'), true), @@ -86,6 +91,68 @@ pub(super) fn parse_descriptor(name: &str) -> Descriptor { } } +/// The marker an advisory queue lane carries in its platform label. +/// +/// A reader of the rendered report, and `xtask release-readiness`, which reads +/// that rendering, tell a lane the queue lets fail from one it does not by it. +pub const ADVISORY_QUEUE_MARKER: &str = "(queue, advisory)"; + +/// The GPU each shared-queue lane stands for, by the lane's name. The queue +/// discovers its lanes from its fleet, so a lane missing from this table is not +/// an error: it renders under its own name in capitals. +const QUEUE_LANE_GPUS: &[(&str, &str)] = &[ + ("mi300x", "MI300X"), + ("mi325x", "MI325X"), + ("mi350p", "MI350P"), + ("mi350x", "MI350X"), + ("mi355x", "MI355X"), + ("r9700s", "R9700"), + ("w7900-dual", "W7900"), +]; + +/// The GPU a shared-queue lane stands for (`w7900-dual` is a W7900), or the +/// lane's own name in capitals for a lane this table does not know yet. +pub fn queue_lane_gpu(lane: &str) -> String { + QUEUE_LANE_GPUS + .iter() + .find(|(name, _)| *name == lane) + .map_or_else(|| lane.to_uppercase(), |(_, gpu)| (*gpu).to_owned()) +} + +/// The artifact name `xtask e2e-queue-import` gives a queue lane's report. +/// +/// For one channel, or none for a lane that reported no channel at all. The lane +/// is named, not the `platform_slug` in its `platform.json`: queue lanes of +/// different models report the same slug (an MI325X lane probes as `mi300x`, an +/// MI355X as `mi350p`), so the slug cannot say which lane a report came from. +/// A lane the queue does not block on carries the `-advisory` suffix. +pub fn queue_artifact_name(lane: &str, advisory: bool, channel: Option<&str>) -> String { + let advisory = if advisory { "-advisory" } else { "" }; + let channel = channel.map_or_else(String::new, |c| format!("-{c}")); + format!("e2e-queue-{lane}{advisory}{channel}-report") +} + +/// Parse the core of a queue lane's artifact name (`mi325x`, `mi325x-advisory`; +/// see [`queue_artifact_name`]). Always Linux: the queue runs Linux pods. +fn queue_descriptor(lane: &str, channel_from_name: Option<&'static str>) -> Descriptor { + let (lane, advisory) = match lane.strip_suffix("-advisory") { + Some(rest) => (rest, true), + None => (lane, false), + }; + let gpu = queue_lane_gpu(lane); + let queue = if advisory { + ADVISORY_QUEUE_MARKER + } else { + "(queue)" + }; + Descriptor { + platform: format!("{gpu} {queue}"), + os: "Linux".to_owned(), + known_bugs: false, + channel_from_name, + } +} + /// The identity-bearing core of an artifact name: `e2e-` prefix and `-report` /// suffix stripped, each relative to the prior result (not `name`), so /// `e2e-report` correctly reduces to `report`, not back to itself. @@ -328,6 +395,33 @@ mod tests { } } + #[test] + fn parse_descriptor_names_each_queue_lane_and_its_gate() { + // Every lane name is a path segment of a queue artifact, so the table is + // what turns `w7900-dual` into the GPU a reader knows. A lane the table + // lacks keeps its own name rather than being guessed at. + for (lane, gpu) in QUEUE_LANE_GPUS { + let blocking = parse_descriptor(&queue_artifact_name(lane, false, Some("release"))); + assert_eq!( + (blocking.platform.as_str(), blocking.os.as_str()), + (format!("{gpu} (queue)").as_str(), "Linux"), + "{lane}" + ); + assert_eq!(blocking.channel_from_name, Some("release")); + let advisory = parse_descriptor(&queue_artifact_name(lane, true, Some("nightly"))); + assert_eq!(advisory.platform, format!("{gpu} {ADVISORY_QUEUE_MARKER}")); + assert_eq!(advisory.channel_from_name, Some("nightly")); + } + let new_lane = parse_descriptor(&queue_artifact_name("mi400x", true, None)); + assert_eq!(new_lane.platform, "MI400X (queue, advisory)"); + assert_eq!(new_lane.channel_from_name, None); + // A queue MI300X lane is not the self-hosted MI300X lane. + assert_ne!( + parse_descriptor("e2e-queue-mi300x-report").platform, + parse_descriptor("e2e-gpu-report").platform + ); + } + #[test] fn parse_descriptor_distinguishes_sdk_version_matrix_legs() { // The sdk_version matrix (ROCMAI-430) uploads non-`current` legs under a diff --git a/crates/e2e-report/src/consolidated/mod.rs b/crates/e2e-report/src/consolidated/mod.rs index 911385029..9a51e9036 100644 --- a/crates/e2e-report/src/consolidated/mod.rs +++ b/crates/e2e-report/src/consolidated/mod.rs @@ -24,7 +24,9 @@ mod sidecars; #[cfg(test)] mod test_support; -pub use descriptor::artifact_name_for_platform_slug; +pub use descriptor::{ + ADVISORY_QUEUE_MARKER, artifact_name_for_platform_slug, queue_artifact_name, queue_lane_gpu, +}; use descriptor::{Descriptor, full_platform_label, parse_descriptor, with_paren_suffix}; use sidecars::{ CommandRecord, ManifestExpectation, PlatformManifest, PlatformVersions, parse_commands, diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index 73a702e74..3eaae3947 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -16,7 +16,10 @@ mod single_report; pub use consolidated::html::{RunMeta, generate_consolidated}; pub use consolidated::markdown::consolidated_summary_markdown; -pub use consolidated::{PlatformReports, artifact_name_for_platform_slug, load_platform_reports}; +pub use consolidated::{ + ADVISORY_QUEUE_MARKER, PlatformReports, artifact_name_for_platform_slug, load_platform_reports, + queue_artifact_name, queue_lane_gpu, +}; pub use parse::{XfailReport, evaluate_xfail, scenario_results_by_id}; pub use single_report::generate; diff --git a/docs/architecture.md b/docs/architecture.md index a303a91f8..f0f8f00d9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -738,6 +738,7 @@ Repository tasks, run through the workspace alias `cargo xtask `: relea | [`demos.rs`](../xtask/src/demos.rs) | `cargo xtask demos`: builds the demo binaries, starts the mock environment before VHS runs, and renders the documentation GIFs. | flat | | [`e2e.rs`](../xtask/src/e2e.rs) | `cargo xtask e2e`: builds release `rocm` and `rocmd` with the E2E test features and runs the cucumber suite, forwarding extra arguments to cucumber. | flat | | [`e2e_prewarm.rs`](../xtask/src/e2e_prewarm.rs) | `cargo xtask e2e-prewarm`: prepares the shared managed runtime the GPU E2E lanes serve against, refreshing it when the channel index publishes something newer or the required packages differ. | flat | +| [`e2e_queue_import.rs`](../xtask/src/e2e_queue_import.rs) | `cargo xtask e2e-queue-import`: puts a finished shared-queue run's per-lane reports where `e2e-report` reads them, named for the lane and the queue's gate, and writes the queue lanes' status section for the top of the support matrix. | flat | | [`e2e_report.rs`](../xtask/src/e2e_report.rs) | `cargo xtask e2e-report`: merges per-platform `report.json` files into one HTML report and a step-summary matrix, one platform per artifact subdirectory. | flat | | [`engine_stdio_client_contract.rs`](../xtask/src/engine_stdio_client_contract.rs) | Source-scan test that only the shared stdio client in [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) puts a request on an engine child's stdin: a file that both pipes a child's stdin and names an engine envelope is reported unless it is a reviewed `ALLOWED` entry, and a stale entry fails too. | flat | | [`env_mutation_contract.rs`](../xtask/src/env_mutation_contract.rs) | Source-scan test that every test which mutates the process environment serializes itself. | flat | diff --git a/xtask/src/main.rs b/xtask/src/main.rs index e3db2768f..4a32095aa 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -22,6 +22,7 @@ mod dash_text_edit_contract; mod demos; mod e2e; mod e2e_prewarm; +mod e2e_queue_import; mod e2e_report; mod engine_stdio_client_contract; mod env_mutation_contract; @@ -331,6 +332,12 @@ pub(crate) enum Command { #[arg(long)] html_out: PathBuf, }, + /// Bring a finished run of the shared GPU queue into the nightly's E2E + /// report: write each lane's reports where `e2e-report` reads them, named + /// for the lane and for whether the queue blocks on it, and write a status + /// section for the top of the support matrix. With `--absent`, import + /// nothing and say why. See docs/nightly-on-the-queue.md. + E2eQueueImport(e2e_queue_import::Args), /// Count received Doctor reports from an export of the report mailbox. /// /// Reads `.eml` files (one message each) and raw `.json` report bodies, @@ -497,6 +504,7 @@ fn run() -> Result<()> { artifacts_dir, html_out, } => e2e_report::run(&artifacts_dir, &html_out)?, + Command::E2eQueueImport(args) => e2e_queue_import::run(&args)?, Command::ReportRollup(args) => report_rollup::run(&args)?, Command::ReleaseReadiness { since_tag, diff --git a/xtask/src/release_readiness.rs b/xtask/src/release_readiness.rs index c3060f557..3cdd834df 100644 --- a/xtask/src/release_readiness.rs +++ b/xtask/src/release_readiness.rs @@ -407,6 +407,11 @@ fn excerpt(text: &str, max_chars: usize) -> String { /// adds for platforms the Total row would otherwise read as having zero /// failures. /// +/// A row for a shared-queue lane the queue does not block on (its platform +/// cell carries [`e2e_report::ADVISORY_QUEUE_MARKER`]) counts for neither: the +/// queue lets that lane fail, so the verdict does too. Its `Fail` cell is +/// taken back out of the Total row, which sums every row. +/// /// Parsing the already-rendered rows, rather than re-deriving per-platform /// pass/fail here, keeps this in lockstep with that crate's known-bugs/xfail /// handling instead of duplicating it. @@ -431,24 +436,31 @@ fn total_e2e_failures(markdown: &str) -> Option { .nth(5) .and_then(|cell| cell.trim().parse::().ok())?; - let silently_empty_platforms = lines[header_idx + 1..total_idx] - .iter() - .filter(|l| { - let t = l.trim_start(); - t.starts_with('|') && !t.starts_with("|---") - }) - .filter(|l| { - let cells: Vec<&str> = l.split('|').collect(); - let status = cells.get(8).map(|c| c.trim()); - let fail = cells - .get(5) - .and_then(|c| c.trim().parse::().ok()) - .unwrap_or(0); - !matches!(status, Some("PASS") | None) && fail == 0 - }) - .count() as u64; + let mut advisory_fails = 0; + let mut silently_empty_platforms = 0; + for line in lines[header_idx + 1..total_idx].iter().filter(|l| { + let t = l.trim_start(); + t.starts_with('|') && !t.starts_with("|---") + }) { + let cells: Vec<&str> = line.split('|').collect(); + let fail = cells + .get(5) + .and_then(|c| c.trim().parse::().ok()) + .unwrap_or(0); + if cells + .get(1) + .is_some_and(|c| c.contains(e2e_report::ADVISORY_QUEUE_MARKER)) + { + advisory_fails += fail; + continue; + } + let status = cells.get(8).map(|c| c.trim()); + if !matches!(status, Some("PASS") | None) && fail == 0 { + silently_empty_platforms += 1; + } + } - Some(total_row_fails + silently_empty_platforms) + Some(total_row_fails.saturating_sub(advisory_fails) + silently_empty_platforms) } /// Neutralize markdown syntax in untrusted text (GitHub issue titles) before @@ -741,6 +753,48 @@ mod tests { assert_eq!(total_e2e_failures(md), Some(1)); } + #[test] + fn total_e2e_failures_leaves_out_the_queue_lanes_the_queue_lets_fail() { + // Real producer output, so the marker the parser looks for is the one the + // report writes: a failing advisory lane and an empty advisory lane add + // nothing, while the same lanes marked blocking add one failure and one + // silent failure. + fn report(status: &str) -> String { + format!( + r#"[{{"name":"F","uri":"f.feature","elements":[{{"name":"s","tags":[],"steps":[{{"keyword":"Given ","name":"x","result":{{"status":"{status}"}}}}]}}]}}]"# + ) + } + let render = |advisory: bool| { + let dir = tempfile::tempdir().expect("tempdir"); + let write = |name: &str, text: &str| { + let path = dir.path().join(name); + std::fs::create_dir_all(&path).expect("mkdir"); + std::fs::write(path.join("report.json"), text).expect("write report"); + (name.to_owned(), path.join("report.json")) + }; + let failing = e2e_report::queue_artifact_name("mi325x", advisory, Some("release")); + let empty = e2e_report::queue_artifact_name("r9700s", advisory, None); + let inputs = vec![ + write("e2e-gpu-report", &report("passed")), + write(&failing, &report("failed")), + write(&empty, "[]"), + ]; + e2e_report::consolidated_summary_markdown(&e2e_report::load_platform_reports(&inputs)) + }; + let advisory = render(true); + assert!( + advisory.contains(e2e_report::ADVISORY_QUEUE_MARKER), + "{advisory}" + ); + assert_eq!(total_e2e_failures(&advisory), Some(0), "{advisory}"); + let blocking = render(false); + assert!( + !blocking.contains(e2e_report::ADVISORY_QUEUE_MARKER), + "{blocking}" + ); + assert_eq!(total_e2e_failures(&blocking), Some(2), "{blocking}"); + } + #[test] fn total_e2e_failures_ignores_the_expectation_grid_and_command_coverage_tables() { // Regression: the scan used to run over every `|`-prefixed line in the From 016c631c3280e360287509eca377912a11c9325c Mon Sep 17 00:00:00 2001 From: Tomas Saaristola <144015745+tomastola@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:11:57 +0200 Subject: [PATCH 3/4] ci: put the queue's nightly lanes in the support matrix and the release-readiness verdict e2e-report-nightly fetches the newest queue-run-results artifact of a main run that finished in the last 24 hours (and that run's job log, for the tested branch and each lane's gate), imports it with `cargo xtask e2e-queue-import` and opens the support matrix with a Queue lanes table. A missing, late or malformed queue run never fails the report: the page says so instead. Retiring the self-hosted R9700, MI350P and W7900 nightly lanes no longer drops them from the E2E-Support-Matrix and Release-Readiness wiki pages, and MI325X, MI350X and MI355X, which only the queue runs, are on them. Signed-off-by: Tomas Saaristola --- .github/workflows/nightly.yml | 91 +++++++++++++++++++++++++++++++++++ docs/ci-hardware-testing.md | 12 +++++ docs/nightly-on-the-queue.md | 75 +++++++++++++++++++++++++++-- 3 files changed, 174 insertions(+), 4 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5399e19e9..4f58c26c4 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1632,12 +1632,103 @@ jobs: pattern: '*-report' path: e2e-artifacts + # The shared GPU queue's nightly (docs/nightly-on-the-queue.md) runs R9700, + # MI350P, W7900 and the Instinct lanes of the queue's fleet on its own + # schedule, and its results are published as one `queue-run-results` + # artifact of that run, not as `*-report` artifacts of this one. So fetch + # the newest such artifact of a run on main that finished, and started, in + # the last 24 hours (a run still going, or one from an earlier night, is + # not today's), and the job log of that run, which is the only place the + # queue prints which branch it tested and which lanes it blocks on. The + # artifact is found by its name, not through the workflow that makes it: + # that workflow runs on a runner no other workflow may name or depend on. + # A conclusion of `failure` is no reason to skip it: a red lane fails the + # queue's run and the artifact is still uploaded. This step never fails the + # job and never stops the report: when there is nothing to import it + # records why, and the next step says so at the top of the page instead of + # leaving the queue's GPUs out without a word. + - name: Fetch the latest queue results + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + run: | + set -uo pipefail + mkdir -p queue-results + absent() { + echo "$1" > queue-results/absent.txt + echo "::warning::No queue lanes in this report: $1" + exit 0 + } + repo_id=$(gh api "repos/${GITHUB_REPOSITORY}" --jq .id) || + absent "the repository could not be read" + since=$(date -u -d '24 hours ago' +%Y-%m-%dT%H:%M:%SZ) + found=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts?name=queue-run-results&per_page=50" | + jq -r --arg since "$since" --argjson repo "$repo_id" ' + [.artifacts[] + | select(.expired == false) + | select(.workflow_run.head_branch == "main" and .workflow_run.head_repository_id == $repo) + | select(.created_at >= $since)] + | sort_by(.created_at) | last // empty | .workflow_run.id') || + absent "the queue results could not be listed" + [ -n "$found" ] || + absent "no queue run on main finished in the last 24 hours (a run still going is not imported)" + # The run of main's workflow can have been asked to test another branch, + # and a run that started a day ago is not tonight's. + started=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${found}" --jq .run_started_at) || + absent "when the newest queue run started could not be read" + started_at=$(date -u -d "$started" +%s 2>/dev/null) && + [ "$started_at" -ge "$(date -u -d '24 hours ago' +%s)" ] || + absent "the newest queue run started more than 24 hours ago" + echo "$started" > queue-results/started.txt + gh run download "$found" --repo "$GITHUB_REPOSITORY" --name queue-run-results --dir queue-results/results || + absent "the queue results of run $found could not be downloaded" + # The log names the branch the run tested and which lanes the queue + # blocks on. Without it nothing is imported. + job=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${found}/jobs" --jq '.jobs[0].id') && + gh api "repos/${GITHUB_REPOSITORY}/actions/jobs/${job}/logs" > queue-results/job.log || + rm -f queue-results/job.log + echo "$found" > queue-results/run-id.txt + + # Turn the run's lanes into reports of this job's own kind, and write the + # status section for the top of the matrix (`cargo xtask e2e-queue-import`). + # A lane the queue marks advisory is labelled so and does not count towards + # the release-readiness verdict; a blocking lane with no result does. + - name: Add the queue lanes to the report + continue-on-error: true + run: | + set -uo pipefail + mkdir -p queue-results + if [ -s queue-results/run-id.txt ] && [ -d queue-results/results ]; then + run_id=$(cat queue-results/run-id.txt) + args=(--queue-dir queue-results/results + --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${run_id}" + --run-started "$(cat queue-results/started.txt 2>/dev/null)") + [ -f queue-results/job.log ] && args+=(--log queue-results/job.log) + else + args=(--absent "$(cat queue-results/absent.txt 2>/dev/null || echo 'fetching the queue results did not complete')") + fi + cargo xtask e2e-queue-import "${args[@]}" \ + --artifacts-dir e2e-artifacts \ + --section-out queue-results/queue-lanes.md || { + # Half an import is worse than none, and none is said at the top of the page. + rm -rf e2e-artifacts/e2e-queue-* queue-results/queue-lanes.md + echo "::warning::The queue lanes could not be added to the report." + cargo xtask e2e-queue-import --absent "the queue lanes could not be imported" \ + --artifacts-dir e2e-artifacts --section-out queue-results/queue-lanes.md || true + } + + # The queue lanes' status leads the page. The matrix keeps its own first + # table, the one `release-readiness` reads its verdict from. - name: Build consolidated report + step summary run: | mkdir -p consolidated cargo xtask e2e-report \ --artifacts-dir e2e-artifacts \ --html-out consolidated/index.html > consolidated/support-matrix.md + if [ -f queue-results/queue-lanes.md ]; then + cat queue-results/queue-lanes.md consolidated/support-matrix.md > "$RUNNER_TEMP/with-queue.md" + mv "$RUNNER_TEMP/with-queue.md" consolidated/support-matrix.md + fi cat consolidated/support-matrix.md >> "$GITHUB_STEP_SUMMARY" # A lane that dies before writing report.json (preflight, build, a crash diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 82f89390b..5b97e3233 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -262,6 +262,18 @@ and GitHub step summary. Since ROCMAI-429 the report keys each column by release/nightly channel matrix renders two columns per platform instead of one channel's result silently overwriting the other. +`e2e-report-nightly` also reads the shared queue's nightly (below), which runs +R9700, MI350P, W7900 and the Instinct lanes of the queue's fleet. It fetches the +`queue-run-results` artifact of the newest queue run that tested `main` and +finished, and started, in the last 24 hours, and `cargo xtask e2e-queue-import` +puts each lane's reports among the others, as `MI325X (queue) Linux (release)` +and the like. A lane the queue does not block on is labelled `(queue, advisory)` +and does not count towards the release-readiness verdict; a blocking lane that +came back with no result is shown as `EMPTY` and does. The page opens with a +`Queue lanes` table. A missing or late queue run never fails the report: the +page says no queue lanes are in it. See +[The support matrix and the wiki](nightly-on-the-queue.md#the-support-matrix-and-the-wiki). + On `main`, the `publish-e2e-wiki` job in `nightly.yml` also copies the consolidated nightly matrix to the `E2E-Support-Matrix` page of the repository wiki, overwriting it each night (the page header names the run, commit and diff --git a/docs/nightly-on-the-queue.md b/docs/nightly-on-the-queue.md index c8a3bdbb6..ff1612566 100644 --- a/docs/nightly-on-the-queue.md +++ b/docs/nightly-on-the-queue.md @@ -135,10 +135,12 @@ model downloads are left to the tools doing them, so a model that downloads more slowly than its scenario waits fails that scenario. The lane fetches only the host's crates, so it refuses to start with `CARGO_BUILD_TARGET` set. -The queue's results are an additional signal, not a replacement: the lanes -write only into the queue artifact, and the `e2e-gpu--report` -artifacts that `nightly.yml`'s consolidated report reads are not produced from -them. +The queue's lanes write only into the queue artifact, and the +`e2e-gpu--report` artifacts that `nightly.yml`'s consolidated report +reads are not produced from them. The report fetches the queue's artifact +itself instead, so the support matrix, the `E2E-Support-Matrix` and +`Release-Readiness` wiki pages and the release-readiness verdict cover the +queue's GPUs too: see [The support matrix and the wiki](#the-support-matrix-and-the-wiki). ## Output @@ -192,6 +194,71 @@ that line is what counts: a blocking lane without a result fails the run. result is not a pass, but they point at queue capacity or infrastructure rather than at a regression. +## The support matrix and the wiki + +`nightly.yml`'s `e2e-report-nightly` job builds the consolidated support +matrix, which `publish-e2e-wiki` publishes as the `E2E-Support-Matrix` wiki page +and `release-readiness-nightly` reads its GO/NO-GO verdict from (see +[ci-hardware-testing.md](ci-hardware-testing.md)). It also reads the queue's +lanes, from the queue's own results, so retiring a self-hosted nightly lane +that the queue covers drops no GPU from either page, and a GPU that only the +queue has (MI325X, MI350X, MI355X) is on them. + +- **Which run.** The newest `queue-run-results` artifact of a run on this + repository's `main` that finished, and started, in the last 24 hours, + whatever the run's conclusion was: a red lane fails the run, and its artifact + is still uploaded. The job finds the artifact by name, not through the + workflow that makes it, because no other workflow may name that one (see + below). The run also has to have tested `main`: a run started from `main` can + be asked to test another branch (the `branch` input), and the job log names + the branch it did. `queue-run.yml` starts at 02:17 UTC and the nightly + starts at 06:00 UTC, its report job after its GPU lanes have finished, so a + run that took under about three and a half hours is in the night's report. A + run that is still going, that started on an earlier day or that tested + another branch is not imported, and neither is a run published again by + `nightly-queue.yml`. +- **Which lanes, and whether each blocks.** The artifact holds the lanes' + files and nothing else about the run, so the job reads the run's job log, whose + lane matrix is the only place the queue prints which lanes it ran and which of + them can fail the run (the `GATE` column). Only the tested branch and commit + and each lane's name, phase, exit status and gate are taken from it, each + checked against a strict pattern first. A lane whose name is not a plain + lowercase word (letters, digits and dashes, and not ending in `-advisory`, + `-release` or `-nightly`) is shown as `unnamed-1` and so on, as a blocking + lane with no result. Without the log nothing is imported. +- **What is imported.** For each lane and channel, + `//results/{report.json,platform.json,commands.jsonl}`, the + three files the report reads, as an artifact named + `e2e-queue-[-advisory]--report`. The `lane.log`, the JUnit and + HTML output and the harness diagnostics are not copied, and nothing the + queue's scan withheld is brought back. The lane's `platform_slug` is replaced + by `queue--`: the slug is the report's column identity, and + queue lanes of different models report the same one (an MI325X lane probes as + `mi300x`, an MI350X or MI355X lane as `mi350p`). The lane command therefore + has to keep writing those files to `/out//results`, as + [`scripts/run-e2e-lane.sh`](../scripts/run-e2e-lane.sh) does. +- **How it reads.** A queue row is labelled `MI325X (queue) Linux (release)`, + so it is not confused with the self-hosted MI300X lane, and a lane the queue + does not block on is `MI325X (queue, advisory) Linux (release)`. The page + opens with a `Queue lanes` table: one row per lane the queue ran, with its + gate, phase, exit status and the channels that reported. +- **What counts towards the verdict.** The same as for the queue's own run. A + blocking lane's failures are in the `E2E` failure count. An advisory lane's are + shown but left out of it. A lane that came back with no result, or whose + channel is missing after it did not finish cleanly, is shown as `EMPTY` and + counts for a blocking lane, because a blocking lane without a result is not + a pass. +- **When there is nothing to import.** No run in the window, a run of another + branch, an artifact or log that cannot be downloaded or an import that fails + never fails the nightly report: the page opens with a `Queue lanes` note + saying that no queue lanes are in it tonight and why, and the rest is built as + before. The verdict does not take that absence for a failure, so read the note + before reading a `GO`. + +What is public follows from the queue's own rules: the artifact is public and +already scanned, only three of its files are used, and the log is the public job +log. + ## What the workflow requires - a self-hosted Linux runner registered with the `queue-report` label alone, From 7288380a28bbd4f6385fc6a8589705a5744cf302 Mon Sep 17 00:00:00 2001 From: Tomas Saaristola <144015745+tomastola@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:12:15 +0200 Subject: [PATCH 4/4] test(workflow-contract): pin how the nightly report imports the queue's lanes The import steps cannot fail the job, fetch by artifact name from a main run of this repository inside the 24 hour window, hold no credential beyond the job's read-only token, and run before the report is built. Signed-off-by: Tomas Saaristola --- xtask/src/workflow_contract.rs | 119 +++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index f0183849b..ccf2cf9cb 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -4399,6 +4399,125 @@ fi } } + /// The nightly report carries the shared queue's GPUs into the matrix, and + /// so into the wiki and the release-readiness verdict, from the queue's own + /// results: the newest finished run on `main`, found by its artifact's name + /// (no workflow may name or wait on the queue's, see + /// `assert_queue_workflow_holds_no_credential`). Neither step may fail the + /// job: a missing or late run is said so at the top of the page. + #[test] + fn nightly_report_imports_the_latest_queue_run_without_depending_on_it() { + let nightly = read_workflow("nightly.yml"); + let job = job_block(&nightly, "e2e-report-nightly"); + let steps = job_steps(job).expect("the report job has steps"); + let position = |prefix: &str| { + steps + .iter() + .position(|(name, _)| name.starts_with(prefix)) + .unwrap_or_else(|| panic!("e2e-report-nightly has a `{prefix}` step")) + }; + let (fetch, add, build) = ( + position("Fetch the latest queue results"), + position("Add the queue lanes to the report"), + position("Build consolidated report"), + ); + assert!( + fetch < add && add < build, + "the queue lanes must be fetched, then put among the reports, before the report is built" + ); + for at in [fetch, add] { + assert_eq!( + step_key(&steps[at].1, "continue-on-error"), + "true", + "`{}` must not be able to fail the nightly report: a missing or late queue run \ + is shown as missing, not as a red report", + steps[at].0 + ); + } + + let fetch = normalized_whitespace(&steps[fetch].1); + for needed in [ + // By artifact name, from a run on this repository's main: the + // workflow that makes it may not be named here. + "actions/artifacts?name=queue-run-results", + ".workflow_run.head_branch == \"main\"", + ".workflow_run.head_repository_id == $repo", + ".expired == false", + // Today's run: not yesterday's, whatever its conclusion was. + "date -u -d '24 hours ago'", + ".created_at >= $since", + "gh run download \"$found\"", + "--name queue-run-results", + // A run that started a day ago is not tonight's, whenever it ended. + "[ \"$started_at\" -ge \"$(date -u -d '24 hours ago' +%s)\" ]", + // The log says which branch the run tested and which lanes block. + "/logs\" > queue-results/job.log", + ] { + assert!( + fetch.contains(needed), + "the queue fetch step must contain `{needed}`:\n{fetch}" + ); + } + assert!( + !fetch.contains("conclusion"), + "a red queue run is still a run: its artifact is uploaded whatever it concluded" + ); + // No credential beyond the job's own read-only token. + let fetch_step = &steps[position("Fetch the latest queue results")].1; + let env = step_env(fetch_step); + assert_eq!( + env.get("GH_TOKEN").map(String::as_str), + Some("${{ github.token }}"), + "{env:?}" + ); + assert_eq!(env.len(), 1, "{env:?}"); + assert!(!fetch_step.contains("secrets."), "{fetch_step}"); + let permissions = job_mapping(job, "permissions"); + assert_eq!( + permissions.get("actions").map(String::as_str), + Some("read"), + "listing and downloading the queue's artifact needs actions: read" + ); + assert_eq!( + permissions.get("contents").map(String::as_str), + Some("read") + ); + assert_eq!(permissions.len(), 2, "{permissions:?}"); + + // Both outcomes go through the importer, which writes the status section. + let add = normalized_whitespace(&steps[position("Add the queue lanes to the report")].1); + for needed in [ + "cargo xtask e2e-queue-import", + "--queue-dir queue-results/results", + "--absent", + "--artifacts-dir e2e-artifacts", + "--section-out queue-results/queue-lanes.md", + // An import that fails leaves no half of one behind, and says so. + "rm -rf e2e-artifacts/e2e-queue-*", + "--absent \"the queue lanes could not be imported\"", + ] { + assert!( + add.contains(needed), + "the queue import step must contain `{needed}`:\n{add}" + ); + } + // The section leads the page; the matrix keeps its own first table. + let build = normalized_whitespace(&steps[build].1); + assert!( + build.contains( + "cat queue-results/queue-lanes.md consolidated/support-matrix.md > \ + \"$RUNNER_TEMP/with-queue.md\"" + ), + "the queue section must be put ahead of the matrix:\n{build}" + ); + // A queue run is never something the report job waits for. + let needs = job_scalar(job, "needs"); + assert!( + !needs.contains("queue"), + "e2e-report-nightly must not depend on the queue's workflows: {needs}" + ); + } + #[test] fn nightly_publishes_the_consolidated_matrix_to_the_wiki() { let nightly = read_workflow("nightly.yml");