diff --git a/crates/e2e-report/src/consolidated/html.rs b/crates/e2e-report/src/consolidated/html.rs new file mode 100644 index 0000000000..474138a072 --- /dev/null +++ b/crates/e2e-report/src/consolidated/html.rs @@ -0,0 +1,451 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! HTML renderer for the consolidated report: [`generate_consolidated`] and the +//! matrix table, legend, expectation grid and per-platform sections it +//! assembles, plus the [`RunMeta`] header line. A child of `consolidated`, so it +//! reads the model types defined there (`PlatformReport`, `Grid`, +//! `CellOutcome`) without those being widened. Re-exported from the crate root +//! by `lib.rs`. + +use std::path::Path; + +use maud::{Markup, html}; + +use super::{CellOutcome, Grid, LEGEND_ENTRIES, PlatformReport, PlatformReports, platform_cell}; +use crate::components::{feature_group, now_utc, page_shell, stats_bar}; + +/// Run-level metadata shown in the report header so a downloaded report can be +/// traced back to the CI run that produced it. All optional — populated from CI +/// env vars, absent for a local run. +#[derive(Default)] +pub struct RunMeta { + pub commit: Option, + pub branch: Option, + pub run_number: Option, + pub event: Option, +} + +impl RunMeta { + fn line(&self) -> Option { + let mut parts = Vec::new(); + if let Some(c) = &self.commit { + parts.push(format!("commit {}", &c[..c.len().min(7)])); + } + if let Some(b) = &self.branch { + parts.push(format!("branch {b}")); + } + if let Some(n) = &self.run_number { + parts.push(format!("run #{n}")); + } + if let Some(e) = &self.event { + parts.push(e.clone()); + } + if parts.is_empty() { + None + } else { + Some(parts.join(" · ")) + } + } +} + +/// Build one consolidated HTML report from a [`PlatformReports`] bundle. +/// +/// The bundle is built from `(label, json_path)` pairs passed to +/// [`load_platform_reports`](super::load_platform_reports); the label +/// identifies the platform/tier (e.g. "GPU Strix Windows (known bugs)"). New +/// platforms need no code change — the caller just passes more inputs there. +/// +/// Reads no input from disk; everything it renders comes from the bundle +/// (see [`load_platform_reports`](super::load_platform_reports)). +pub fn generate_consolidated( + bundle: &PlatformReports, + html_out: &Path, + meta: &RunMeta, +) -> std::io::Result<()> { + let reports = &bundle.reports; + let now = now_utc(); + let all_ok = reports.iter().all(PlatformReport::ok); + let overall = if reports.is_empty() { + ("status-fail", "No platform reports found".to_string()) + } else if all_ok { + ("status-pass", "All platforms in expected state".to_string()) + } else { + let bad = reports.iter().filter(|r| !r.ok()).count(); + ("status-fail", format!("{bad} platform(s) need attention")) + }; + + let body = html! { + div.header { + h1 { "Consolidated E2E Report" } + div.generated { + @if let Some(line) = meta.line() { (line) br; } + "Generated " (now) + } + } + + h2 { "Summary Information" } + table.summary-table { + tr { + td { "Status:" } + td class=(overall.0) { (overall.1) } + } + tr { td { "Rows:" } td { (reports.len()) } } + } + + @if reports.is_empty() { + p { "No per-platform report.json files were found to consolidate." } + } @else { + h2 { "Platforms" } + (matrix_table(reports)) + (legend()) + + (expectation_grid_html(&bundle.grid)) + + h2 { "Per-platform Details" } + div.details { + @for report in reports { + (platform_section(report)) + } + } + } + }; + let markup = page_shell("Consolidated E2E Report", body); + + std::fs::write(html_out, markup.into_string()) +} + +/// Render the reconciled expectation grid as an HTML section (for the standalone +/// report). Empty markup only when there is truly nothing to report — no +/// column/row AND no excluded platform to caption (see +/// [`Grid::nothing_to_report`]); a report where every platform lacks a usable +/// `platform.json` still renders a heading and caption naming them. +pub(super) fn expectation_grid_html(grid: &Grid) -> Markup { + // Not `grid.is_empty()`: a report where every platform lacks a + // `platform.json` has no columns/groups either, but still has an + // `excluded_note` worth showing — otherwise that case regresses to a + // silently missing section, the exact complaint issue #702 raises (just + // for every platform at once instead of some of them). + if grid.nothing_to_report() { + return html! {}; + } + let problems = grid.problems(); + let excluded_note = grid.excluded_note(); + html! { + h2 { "Expectation grid (scenario × platform)" } + @if !grid.is_empty() { + p.grid-legend { + @for (i, (outcome, text)) in LEGEND_ENTRIES.iter().copied().enumerate() { + @if i > 0 { " · " } + @if outcome.grid_class().is_empty() { + (outcome.glyph()) + } @else { + span class=(outcome.grid_class()) { (outcome.glyph()) } + } + " " (text) + } + } + } + @if let Some(note) = &excluded_note { + p.empty-note { (note) } + } + // One table per feature, so a reader can scan a single area of the CLI + // instead of one undivided 60-row block. + @for group in &grid.groups { + h3.feature-heading { (group.feature) } + table.stats { + thead { + tr { + th { "Scenario" } + @for col in &grid.columns { + @let versions = col.versions.summary(); + th { + (col.label()) + @if !col.engine.is_empty() { br; small { (col.engine) } } + @if !versions.is_empty() { br; small.versions { (versions) } } + } + } + } + } + tbody { + @for row in &group.rows { + tr { + td { + @if !row.name.is_empty() { (row.name) br; } + code { (row.id) } + } + @for col in &grid.columns { + @let outcome = col.outcomes.get(&row.id).copied().unwrap_or(CellOutcome::Missing); + // One combined class attr — `td.num class=(..)` would emit + // two `class` attributes, and the browser keeps only the + // first ("num"), dropping the status colour. + td class=(format!("num {}", outcome.grid_class())) { + (outcome.glyph()) + } + } + } + } + } + } + } + @if !problems.is_empty() { + h3 { "Needs attention" } + ul { + @for (platform_label, id, outcome, detail) in &problems { + li { + b { + @match outcome { + CellOutcome::Xpass => "XPASS", + CellOutcome::UnexpectedFail => "unexpected failure", + CellOutcome::RanWhenNa => "ran despite n/a", + _ => "issue", + } + } + " on " code { (platform_label) } ": " code { (id) } + @if let Some(d) = detail { + @if let Some(b) = &d.bug { " (" (b) ")" } + @if !d.effective_engine.is_empty() { " [engine: " (d.effective_engine) "]" } + @if let Some(r) = &d.reason { " — " (r) } + } + } + } + } + } + } +} + +fn matrix_table(reports: &[PlatformReport]) -> Markup { + let (mut t_total, mut t_pass, mut t_fail, mut t_skip, mut t_xfail) = (0, 0, 0, 0, 0); + for r in reports { + let (total, pass, fail, skip, xf) = r.display_counts(); + t_total += total; + t_pass += pass; + t_fail += fail; + t_skip += skip; + t_xfail += xf; + } + html! { + table.stats { + tr { + th { "Platform" } th { "OS" } + th { "Total" } th { "Pass" } th { "Fail" } th { "Skip" } + th { "Xfail" } th { "Status" } + th { "Pass / Fail / Skip" } + } + @for r in reports { + @let (total, pass, fail, skip, xf) = r.display_counts(); + tr { + td { (platform_cell(r)) } + td { (r.desc.os) } + td.num { (total) } + td.num { (pass) } + td.num { (fail) } + td.num { (skip) } + td.num { @if xf > 0 || r.is_known_bugs { (xf) } @else { "n/a" } } + td class=(if r.ok() { "status-pass" } else { "status-fail" }) { + (r.status_text()) + } + td { (stats_bar(&r.stats)) } + } + } + tr.total-row { + td { "Total" } td {} + td.num { (t_total) } + td.num { (t_pass) } + td.num { (t_fail) } + td.num { (t_skip) } + td.num { (t_xfail) } + td {} td {} + } + } + } +} + +/// Explain the non-obvious columns/terms so the report is self-contained. +fn legend() -> Markup { + html! { + div.legend { + h3 { "Legend" } + ul { + li { + b { "Mock" } + " — no GPU. The CLI runs against a fake in-process model " + "server (a planted service record), validating CLI behaviour " + "and wiring without hardware. Runs on a GitHub-hosted runner " + "and gates the PR." + } + li { + b { "Every other platform" } + " — real self-hosted GPU hardware; non-blocking while proven out." + } + li { + b { "Status" } + " — PASS when the platform is in its expected state (every " + "expect-pass scenario passed and every known bug failed as " + "expected); FAIL on an unexpected failure or an XPASS (a known " + "bug that unexpectedly passed — remove its @expected-failure tag)." + } + li { b { "Xfail" } " — count of known-bug scenarios that failed as expected." } + li { b { "Skip" } " — scenarios not applicable on this platform (not run)." } + } + } + } +} + +fn platform_section(report: &PlatformReport) -> Markup { + let badge_class = if report.ok() { + "badge-pass" + } else { + "badge-fail" + }; + html! { + details.platform open[!report.ok()] { + summary.platform-row { + span class=(format!("badge {badge_class}")) { (report.status_text()) } + span.platform-name { (report.label) } + span.elapsed { (report.stats.total) " scenarios" } + } + @if report.features.is_empty() { + p.empty-note { "No report.json data for this platform." } + } @else { + @for feature in &report.features { + (feature_group(feature)) + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::consolidated::test_support::{feature_json, write_platform}; + use crate::consolidated::{load_platform_reports, load_reports}; + + #[test] + fn generate_consolidated_renders_the_expectation_grid() { + // Review finding (6f130efe round, back when `generate_consolidated` + // was `generate_consolidated_from_reports` and took `reports` and + // `inputs` as two loose parameters): passing `&[]` instead of `inputs` + // to the `expectation_grid_html(inputs)` call inside passed every + // existing test at the time, because the fixtures used elsewhere had + // no platform.json, so the grid was empty either way. The same slip + // now would be building the bundle's grid from `&[]` in + // `load_platform_reports`. Give this one a platform.json via + // write_platform and assert the grid heading actually renders. + let report = feature_json(&[(&["id:serve-a"], &["passed"])]); + let platform = r#"{ + "platform_slug": "mock", + "capability": {"effective_serve_engine": "none"}, + "expectations": [ + {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"pass"} + ] + }"#; + let (_d, path) = write_platform(&report, platform); + let inputs = vec![("mock".to_string(), path)]; + let bundle = load_platform_reports(&inputs); + + let out = tempfile::NamedTempFile::new().expect("temp"); + generate_consolidated(&bundle, out.path(), &RunMeta::default()).expect("generate"); + let html = std::fs::read_to_string(out.path()).expect("read"); + assert!( + html.contains("Expectation grid (scenario × platform)"), + "grid heading missing from output:\n{html}" + ); + } + + #[test] + fn generate_consolidated_writes_html() { + // Same platform_slug, two channels: the HTML matrix (matrix_table) must + // carry the channel suffix too, not just its markdown twin. + let release_platform = r#"{ + "platform_slug": "mi300x", + "capability": {"effective_serve_engine": "vllm"}, + "versions": {"channel": "release"} + }"#; + let nightly_platform = r#"{ + "platform_slug": "mi300x", + "capability": {"effective_serve_engine": "vllm"}, + "versions": {"channel": "nightly"} + }"#; + let report = feature_json(&[(&[], &["passed"])]); + let (_d1, release_path) = write_platform(&report, release_platform); + let (_d2, nightly_path) = write_platform(&report, nightly_platform); + let out = tempfile::NamedTempFile::new().expect("temp"); + let inputs = vec![ + ("mi300x-release".to_string(), release_path), + ("mi300x-nightly".to_string(), nightly_path), + ]; + generate_consolidated( + &load_platform_reports(&inputs), + out.path(), + &RunMeta::default(), + ) + .expect("generate"); + let html = std::fs::read_to_string(out.path()).expect("read"); + assert!(html.contains("Consolidated E2E Report")); + assert!( + html.contains("Consolidated E2E Report"), + "the page_shell title argument must reach the tag, not just the <h1>" + ); + assert!(html.contains("Platforms")); + assert!(html.contains("Legend")); + assert!( + html.contains("Mi300x (release)"), + "HTML matrix must carry the release channel suffix" + ); + assert!( + html.contains("Mi300x (nightly)"), + "HTML matrix must carry the nightly channel suffix" + ); + + // The sort key's channel element only matters if it actually drives + // order: nightly's lexical head start ("n" < "r") must place it before + // release in the rendered matrix, or this element of the tuple is dead. + let nightly_pos = html.find("Mi300x (nightly)").expect("nightly cell present"); + let release_pos = html.find("Mi300x (release)").expect("release cell present"); + assert!( + nightly_pos < release_pos, + "HTML matrix must order nightly before release for the same platform" + ); + } + + #[test] + fn expectation_grid_html_legend_colors_every_red_glyph() { + // Every outcome that renders red in the grid (`grid_class() == + // "status-fail"`) must also render red in the legend, not just the + // original Xfail/UnexpectedFail pair — Xpass, RanWhenNa, and Absent + // are equally `status-fail` in the grid and were previously left as + // plain, uncoloured glyphs in the legend. + let report = feature_json(&[(&["id:serve-x"], &["passed"])]); + let platform = r#"{ + "platform_slug": "mi300x", + "capability": {"effective_serve_engine": "vllm"}, + "expectations": [ + {"id":"serve-x","effective_engine":"vllm","expected":"pass"} + ] + }"#; + let (_d, path) = write_platform(&report, platform); + let inputs = vec![("mi300x".to_string(), path)]; + + let html = expectation_grid_html(&Grid::build(&load_reports(&inputs))).into_string(); + for glyph in ["⚠️XPASS", "⚠️n/a-ran", "⚠️no-result", "❌FAIL"] { + let needle = format!("class=\"status-fail\">{glyph}</span>"); + assert!( + html.contains(&needle), + "legend must colour {glyph:?} status-fail:\n{html}" + ); + } + // The known-bug xfail glyph is styled grey (status-xfail), not red. + assert!( + html.contains("class=\"status-xfail\">✗</span>"), + "legend must colour the xfail glyph status-xfail:\n{html}" + ); + // Genuinely unstyled outcomes stay bare glyphs, not empty-class spans. + assert!( + !html.contains("class=\"\">"), + "unstyled legend glyphs must not be wrapped in an empty-class span:\n{html}" + ); + } +} diff --git a/crates/e2e-report/src/consolidated/markdown.rs b/crates/e2e-report/src/consolidated/markdown.rs new file mode 100644 index 0000000000..798a04914e --- /dev/null +++ b/crates/e2e-report/src/consolidated/markdown.rs @@ -0,0 +1,780 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Markdown renderer for the consolidated report: [`consolidated_summary_markdown`] +//! (the GitHub-flavoured table piped into `$GITHUB_STEP_SUMMARY`), the +//! expectation grid and the Scenario reference section it appends. A child of +//! `consolidated`, so it reads the model types defined there (`PlatformReport`, +//! `Grid`, `CellOutcome`) without those being widened, and it pulls in the +//! command-coverage table from `command_coverage.rs`. Re-exported from the +//! crate root by `lib.rs`. + +use super::{CellOutcome, Grid, LEGEND_ENTRIES, PlatformReports, ScenarioReference, platform_cell}; +use crate::command_coverage::command_coverage_markdown; + +/// Render the consolidated result from a [`PlatformReports`] bundle as a +/// GitHub-flavoured markdown table, for piping into `$GITHUB_STEP_SUMMARY`. +/// Same bundle as [`generate_consolidated`]. +/// +/// Reads nothing from disk; everything it renders comes from the bundle +/// (see [`load_platform_reports`]). +/// +/// [`generate_consolidated`]: super::html::generate_consolidated +/// [`load_platform_reports`]: super::load_platform_reports +pub fn consolidated_summary_markdown(bundle: &PlatformReports) -> String { + use std::fmt::Write as _; + + let reports = &bundle.reports; + let mut out = String::from("## E2E consolidated report\n\n"); + if reports.is_empty() { + out.push_str("_No per-platform report.json files were found to consolidate._\n"); + return out; + } + + out.push_str("| Platform | OS | Total | Pass | Fail | Skip | Xfail | Status |\n"); + out.push_str("|---|---|--:|--:|--:|--:|--:|:--|\n"); + let (mut t_total, mut t_pass, mut t_fail, mut t_skip, mut t_xfail) = (0, 0, 0, 0, 0); + for r in reports { + let (total, pass, fail, skip, xf) = r.display_counts(); + // Xfail column only applies where there are known bugs to invert; a plain + // expect-pass platform (no xfail entries) shows N/A. + let xfail = if xf > 0 || r.is_known_bugs { + xf.to_string() + } else { + "n/a".to_string() + }; + t_total += total; + t_pass += pass; + t_fail += fail; + t_skip += skip; + t_xfail += xf; + // Component versions in the Platform/OS cells: ROCm/vLLM/lemonade under the + // platform, the OS version under the OS. Absent components are omitted (mock + // has no runtime; a not-yet-probed source is simply skipped). + let plat_base = platform_cell(r); + let plat_cell = match r.versions.platform_stack() { + s if s.is_empty() => plat_base, + s => format!("{plat_base}<br><sub>{s}</sub>"), + }; + let os_cell = match r.versions.os.as_deref() { + Some(v) if v != r.desc.os => format!("{}<br><sub>{}</sub>", r.desc.os, v), + _ => r.desc.os.clone(), + }; + // `writeln!` into a String never fails; the discard keeps clippy happy. + let _ = writeln!( + out, + "| {} | {} | {} | {} | {} | {} | {} | {} |", + plat_cell, + os_cell, + total, + pass, + fail, + skip, + xfail, + r.status_text(), + ); + } + let _ = writeln!( + out, + "| **Total** | | {t_total} | {t_pass} | {t_fail} | {t_skip} | {t_xfail} | |", + ); + + out.push_str( + "\n**Mock** runs the real `rocm` CLI, but fakes the inference backend: instead \ + of downloading a runtime and launching a real engine (vLLM/lemonade) on a \ + GPU, a tiny in-process HTTP server stands in for the OpenAI-compatible \ + endpoint that engine would expose (`/v1/models`, `/v1/chat/completions`). \ + This exercises CLI behaviour — alias resolution, service discovery, chat \ + forwarding — with no GPU, no model download, and no engine process, so it \ + runs on a GitHub-hosted runner. It **gates the PR**: it runs on every push, \ + and if it fails the PR's required check goes red and the PR cannot merge. \ + **Every other platform** is a real self-hosted GPU host running real \ + engines — the rows below are the list, so naming them here would only go \ + stale as lanes are added. They are **non-blocking**: they still run and \ + are reported here, but \ + a failure does NOT block the PR from merging (the hardware/runners are still \ + being proven out, so their results are informational rather than a merge \ + gate).\n\n\ + Column meanings: **Pass** = scenarios that passed as expected; \ + **Xfail** = known bugs that failed as expected (healthy — the bug still \ + reproduces); **Skip** = not applicable on this platform (e.g. a GPU-only \ + scenario on Mock); **Fail** = unexpected — a scenario that should pass but \ + failed, or a known bug that unexpectedly passed (its tag is now stale). \ + **Status** is PASS unless a platform has any Fail.\n", + ); + + // Call out anything that needs a human: XPASS (fixed bug, stale tag) and + // untagged failures in a known-bugs run. + let mut notes = Vec::new(); + for r in reports { + for name in &r.xfail.xpass { + notes.push(format!( + "- **XPASS** in _{}_: `{}` is tagged `@expected-failure` but passed — remove the tag.", + r.label, name + )); + } + for name in &r.xfail.untagged_failures { + if r.is_known_bugs { + notes.push(format!( + "- **Regression** in _{}_: `{}` failed but is not tagged `@expected-failure`.", + r.label, name + )); + } + } + } + if !notes.is_empty() { + out.push_str("\n### Needs attention\n\n"); + for n in notes { + out.push_str(&n); + out.push('\n'); + } + } + + // The per-(scenario × platform) expectation grid, when platform.json + // sidecars are present (the new expectation system). Placed before the + // command-coverage table; it supersedes the coarse platform×tier matrix for + // "where should each test pass / not matter / not run". The scenario + // map supplies names for the grid's Scenario column (name + id link). + out.push_str(&expectation_grid_markdown(&bundle.grid, &bundle.scenarios)); + + out.push_str(&command_coverage_markdown(reports)); + + // Scenario reference LAST (after command coverage): each id's actual Gherkin + // scenario, anchored so the grid's id links resolve here. + out.push_str(&scenario_reference_markdown( + &bundle.grid, + &bundle.scenarios, + )); + + out +} + +/// Render the reconciled (scenario-id × platform) expectation grid as markdown, +/// plus a "needs attention" list of every XPASS / unexpected-fail / ran-when-NA. +/// Empty string only when there is truly nothing to report — no column/row +/// AND no excluded platform to caption (see [`Grid::nothing_to_report`]); a +/// report where every platform lacks a usable `platform.json` still renders a +/// heading and caption naming them. +fn expectation_grid_markdown(grid: &Grid, scenarios: &ScenarioReference) -> String { + use std::fmt::Write as _; + + // Not `grid.is_empty()`: see the matching comment in `expectation_grid_html` + // — a report where every platform lacks a `platform.json` still has an + // `excluded_note` worth showing, even with no columns/groups to table. + if grid.nothing_to_report() { + return String::new(); + } + + let mut out = String::from("\n### Expectation grid (scenario × platform)\n\n"); + if !grid.is_empty() { + let legend = LEGEND_ENTRIES + .iter() + .map(|(outcome, text)| format!("{} {text}", outcome.glyph())) + .collect::<Vec<_>>() + .join(" · "); + let _ = writeln!(out, "_{legend}_\n"); + } + if let Some(note) = grid.excluded_note() { + let _ = writeln!(out, "_{note}_\n"); + } + + // One table per feature, under its own heading — a single undivided table of + // every scenario in the suite is unreadable, and gives no clue where one area + // of the CLI ends and the next begins. + for group in &grid.groups { + let _ = writeln!(out, "#### {}\n", group.feature); + + // Header: one column per platform, with its effective engine. Component + // versions live in the summary matrix above, not here. + out.push_str("| Scenario |"); + for col in &grid.columns { + let eng = if col.engine.is_empty() { + String::new() + } else { + format!("<br><sub>{}</sub>", col.engine) + }; + let _ = write!(out, " {}{} |", col.label(), eng); + } + out.push('\n'); + out.push_str("|---|"); + for _ in &grid.columns { + out.push_str(":--:|"); + } + out.push('\n'); + + for row in &group.rows { + // Scenario cell: human name on top, the @id below as a link to its + // entry in the Scenario reference section (GitHub anchors + // `##### <id>` to `#<id>`). Prefer the name recorded in platform.json + // — it covers a scenario that was skipped everywhere and so has no + // report.json entry to read a name from. + let name = if row.name.is_empty() { + scenarios.get(&row.id).map_or("", |(n, _)| n.as_str()) + } else { + row.name.as_str() + }; + let id = &row.id; + let _ = write!(out, "| {name}<br>[`{id}`](#{id}) |"); + for col in &grid.columns { + let g = col + .outcomes + .get(id) + .copied() + .unwrap_or(CellOutcome::Missing); + let _ = write!(out, " {} |", g.glyph()); + } + out.push('\n'); + } + out.push('\n'); + } + + // Needs-attention list from reconciliation problems. + let problems = grid.problems(); + if !problems.is_empty() { + out.push_str("\n### Needs attention\n\n"); + for (platform_label, id, outcome, detail) in problems { + let bug = detail + .and_then(|d| d.bug.as_deref()) + .map(|b| format!(" ({b})")) + .unwrap_or_default(); + let engine = detail + .map(|d| d.effective_engine.as_str()) + .filter(|e| !e.is_empty()) + .map(|e| format!(" [engine: {e}]")) + .unwrap_or_default(); + let reason = detail + .and_then(|d| d.reason.as_deref()) + .map(|r| format!(" — {r}")) + .unwrap_or_default(); + let kind = match outcome { + CellOutcome::Xpass => "XPASS", + CellOutcome::UnexpectedFail => "unexpected failure", + CellOutcome::RanWhenNa => "ran despite n/a", + _ => "issue", + }; + let _ = writeln!( + out, + "- **{kind}** on `{platform_label}`: `{id}`{bug}{engine}{reason}" + ); + } + } + + out +} + +/// Render the Scenario reference section: each scenario `@id` with its actual +/// Gherkin scenario (name + steps), anchored by the id (`##### <id>` → GitHub +/// anchor `#<id>`) so the grid's id links resolve. Grouped by feature and ordered +/// like the grid. Empty when no expectation grid exists. +fn scenario_reference_markdown(grid: &Grid, scenarios: &ScenarioReference) -> String { + use std::fmt::Write as _; + + // Only emit when there is a grid to reference (platform.json sidecars present), + // matching where the id links are generated. + if grid.is_empty() { + return String::new(); + } + + // Walk the grid's own order so the reference is laid out feature by feature, + // matching the tables that link into it. Every grid row gets an entry — a + // scenario skipped on every platform has no steps to show, but still needs + // its anchor or the grid's link to it dangles. + // + // Scoped to grid rows deliberately: this section exists to back the grid's + // links, so it documents exactly what the grid shows. A scenario present only + // in an artifact with no `platform.json` sidecar has no grid row and so gets + // no entry — nothing links to it either. + let mut out = String::from("\n### Scenario reference\n\n"); + for group in &grid.groups { + let _ = writeln!(out, "#### {}\n", group.feature); + for row in &group.rows { + let entry = scenarios.get(&row.id); + // Prefer the platform.json name, as the grid does — it is the only + // source that covers a scenario which ran nowhere. + let name = if row.name.is_empty() { + entry.map_or("", |(n, _)| n.as_str()) + } else { + row.name.as_str() + }; + let _ = writeln!(out, "##### {}\n", row.id); + if !name.is_empty() { + let _ = writeln!(out, "_{name}_\n"); + } + match entry { + Some((_, steps)) => { + for step in steps { + let _ = writeln!(out, "- {step}"); + } + } + // Ran nowhere (n/a on every platform), so no steps were recorded. + None => out.push_str("_Not run on any platform in this run._\n"), + } + out.push('\n'); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::consolidated::load_platform_reports; + use crate::consolidated::test_support::{ + feature_json, grid_order, write_platform, write_report, + }; + + #[test] + fn consolidated_summary_markdown_has_a_row_per_platform() { + let a = write_report(&feature_json(&[(&[], &["passed"]), (&[], &["passed"])])); + let b = write_report(&feature_json(&[(&["expected-failure"], &["failed"])])); + let inputs = vec![ + ("e2e-report".to_string(), a.path().to_path_buf()), + ( + "e2e-gpu-known-bugs-report".to_string(), + b.path().to_path_buf(), + ), + ]; + let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); + assert!(md.contains("| Mock | Linux | 2 | 2 | 0 | 0 | n/a | PASS |")); + assert!(md.contains("| MI300X | Linux | 1 | 0 | 1 | 0 | 1 | PASS |")); + } + + #[test] + fn consolidated_summary_markdown_flags_xpass() { + let b = write_report(&feature_json(&[(&["expected-failure"], &["passed"])])); + let inputs = vec![( + "e2e-gpu-known-bugs-report".to_string(), + b.path().to_path_buf(), + )]; + let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); + assert!(md.contains("Needs attention")); + assert!(md.contains("XPASS")); + } + + #[test] + fn consolidated_summary_markdown_empty_inputs() { + let md = consolidated_summary_markdown(&load_platform_reports(&[])); + assert!(md.contains("No per-platform report.json files")); + } + + #[test] + fn grid_groups_by_feature_and_orders_by_index() { + // Declared deliberately out of order (and interleaved across features) + // so a passing assertion can only come from real grouping + sorting, + // not from the input order. + let report = feature_json(&[ + (&["id:serve-b"], &["passed"]), + (&["id:examine-a"], &["passed"]), + ]); + let platform = r#"{ + "platform_slug": "mock", + "capability": {"effective_serve_engine": "none"}, + "expectations": [ + {"id":"serve-b","feature":"Serving","scenario":"serve-02 - second","expected":"pass"}, + {"id":"examine-a","feature":"Examine","scenario":"examine-09 - ninth","expected":"pass"}, + {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"skip"}, + {"id":"examine-b","feature":"Examine","scenario":"examine-10 - tenth","expected":"skip"} + ] + }"#; + let (_d, path) = write_platform(&report, platform); + let inputs = vec![("mock".to_string(), path)]; + + assert_eq!( + grid_order(&inputs), + vec![ + ("Examine".to_string(), "examine-a".to_string()), + ("Examine".to_string(), "examine-b".to_string()), + ("Serving".to_string(), "serve-a".to_string()), + ("Serving".to_string(), "serve-b".to_string()), + ], + "rows must group by feature and sort by index (09 before 10, not \ + lexically), regardless of declaration order", + ); + + // Each feature gets its own table under its own heading. + let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); + assert!( + md.contains("#### Examine"), + "missing feature heading:\n{md}" + ); + assert!( + md.contains("#### Serving"), + "missing feature heading:\n{md}" + ); + // The human name is shown alongside the id, sourced from platform.json — + // serve-a was skipped everywhere, so report.json has no name for it. + assert!( + md.contains("serve-01 - first<br>[`serve-a`](#serve-a)"), + "skipped scenario should still show its name:\n{md}" + ); + } + + #[test] + fn consolidated_summary_markdown_orders_rows_by_report_sort_key() { + // report_sort_key_orders_by_every_dimension already pins all four + // tuple elements via a direct call to report_sort_key with synthetic + // PlatformReports. This test instead goes through the public + // consolidated_summary_markdown entry point with realistic + // artifact-name fixtures, so a regression in load_platform_reports' + // use of report_sort_key — e.g. calling it with the wrong field order, + // or not calling it at all — would still be caught even if + // report_sort_key itself stayed correct in isolation. + // + // Pins all four elements of the sort-key tuple, in precedence order: + // platform, os, known_bugs, channel. Three Strix Halo rows alone + // can't pin `platform` — they all share it, so dropping or swapping + // that element would not change their order — so a fourth row on a + // different platform ("Unknown"/"Unknown", via the explicit + // "e2e-unknown-report" artifact name) is added: "Strix Halo" sorts + // before "Unknown" by platform, but "Unknown" alone (as an OS) sorts + // before "Windows", so dropping the platform element from the key, or + // swapping it with os, both move this row ahead of the Windows row + // instead of after it. Within the Strix Halo rows: "Strix Halo"/Ubuntu + // before "Strix Halo"/Windows pins os; the two Ubuntu rows, identical + // except for the known-bugs artifact suffix, pin known_bugs ordering + // (false < true) once os and channel are held equal. A fifth and + // sixth row on two more distinct platforms with opposite channels — + // MI300X/release and R9700/nightly — pin channel's precedence below + // platform: "nightly" sorts before "release", so if channel outranked + // platform in the key, R9700 would sort first; "MI300X" sorts before + // "R9700" by platform, so the correct key keeps MI300X first + // regardless. A seventh row, same platform/os as the MI300X row but + // known_bugs=true and channel=nightly, pins that known_bugs outranks + // channel too: "nightly" sorts before "release", so if channel + // outranked known_bugs, this row would sort before the MI300X/release + // row despite known_bugs=false < true. Channel's own effect on + // ordering — as a tie-breaker once platform/os/known_bugs are equal — + // is pinned by generate_consolidated_writes_html and + // command_coverage_distinguishes_channels_on_same_platform, not + // repeated here. + // No platform.json sidecar, so display_counts falls back to raw junit + // stats and each row's distinct scenario count (1/2/3/4/5/6/7) tells + // the rows apart in the rendered table. + let ubuntu_plain_report = write_report(&feature_json(&[(&[], &["passed"])])); + let ubuntu_known_bugs_report = + write_report(&feature_json(&[(&[], &["passed"]), (&[], &["passed"])])); + let windows_plain_report = write_report(&feature_json(&[ + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + ])); + let unknown_report = write_report(&feature_json(&[ + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + ])); + let mi300x_release_report = write_report(&feature_json(&[ + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + ])); + let r9700_nightly_report = write_report(&feature_json(&[ + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + ])); + let mi300x_known_bugs_nightly_report = write_report(&feature_json(&[ + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + (&[], &["passed"]), + ])); + let inputs = vec![ + ( + "e2e-gpu-strix-windows-report".to_string(), + windows_plain_report.path().to_path_buf(), + ), + ( + "e2e-gpu-strix-ubuntu-known-bugs-report".to_string(), + ubuntu_known_bugs_report.path().to_path_buf(), + ), + ( + "e2e-gpu-strix-ubuntu-report".to_string(), + ubuntu_plain_report.path().to_path_buf(), + ), + ( + "e2e-unknown-report".to_string(), + unknown_report.path().to_path_buf(), + ), + ( + "e2e-gpu-release-report".to_string(), + mi300x_release_report.path().to_path_buf(), + ), + ( + "e2e-gpu-rad3-nightly-report".to_string(), + r9700_nightly_report.path().to_path_buf(), + ), + ( + "e2e-gpu-known-bugs-nightly-report".to_string(), + mi300x_known_bugs_nightly_report.path().to_path_buf(), + ), + ]; + + let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); + let ubuntu_plain_pos = md + .find("| Strix Halo | Ubuntu | 1 |") + .expect("plain Ubuntu row"); + let ubuntu_known_bugs_pos = md + .find("| Strix Halo | Ubuntu | 2 |") + .expect("known-bugs Ubuntu row"); + let windows_pos = md + .find("| Strix Halo | Windows | 3 |") + .expect("Windows row"); + let unknown_pos = md + .find("| Unknown | Unknown | 4 |") + .expect("Unknown-platform row"); + let mi300x_pos = md + .find("| MI300X (release) | Linux | 5 |") + .expect("MI300X release row"); + let r9700_pos = md + .find("| R9700 (nightly) | Linux | 6 |") + .expect("R9700 nightly row"); + let mi300x_known_bugs_nightly_pos = md + .find("| MI300X (nightly) | Linux | 7 |") + .expect("MI300X known-bugs nightly row"); + assert!( + ubuntu_plain_pos < ubuntu_known_bugs_pos, + "same platform/os: known_bugs=false must sort before known_bugs=true:\n{md}" + ); + assert!( + ubuntu_known_bugs_pos < windows_pos, + "same platform: Ubuntu must sort before Windows regardless of known_bugs:\n{md}" + ); + assert!( + windows_pos < unknown_pos, + "\"Strix Halo\" must sort before \"Unknown\" by platform, even though \"Unknown\" as an OS sorts before \"Windows\":\n{md}" + ); + assert!( + mi300x_pos < r9700_pos, + "\"MI300X\" must sort before \"R9700\" by platform, even though its channel \ + (\"release\") sorts after R9700's (\"nightly\"):\n{md}" + ); + assert!( + mi300x_pos < mi300x_known_bugs_nightly_pos, + "same platform/os: known_bugs=false must sort before known_bugs=true, even though \ + the known_bugs=true row's channel (\"nightly\") sorts before the known_bugs=false \ + row's (\"release\"):\n{md}" + ); + } + + #[test] + fn scenario_reference_anchors_every_grid_row() { + // A scenario that is n/a everywhere has no report.json entry — it still + // needs an anchor, or the grid's link to it dangles. + let report = feature_json(&[(&["id:serve-x"], &["passed"])]); + let platform = r#"{ + "platform_slug": "mock", + "capability": {"effective_serve_engine": "none"}, + "expectations": [ + {"id":"serve-x","feature":"Serving","scenario":"serve-01 - ran","expected":"pass"}, + {"id":"serve-z","feature":"Serving","scenario":"serve-02 - skipped","expected":"skip"} + ] + }"#; + let (_d, path) = write_platform(&report, platform); + let md = + consolidated_summary_markdown(&load_platform_reports(&[("mock".to_string(), path)])); + assert!(md.contains("##### serve-x"), "missing anchor:\n{md}"); + assert!( + md.contains("##### serve-z"), + "a never-run scenario still needs its anchor:\n{md}" + ); + assert!( + md.contains("_Not run on any platform in this run._"), + "a never-run scenario should say so instead of showing no steps:\n{md}" + ); + } + + #[test] + fn scenario_reference_anchors_grid_when_report_has_no_scenarios() { + let report = feature_json(&[]); + let platform = r#"{ + "platform_slug": "mock", + "capability": {"effective_serve_engine": "none"}, + "expectations": [ + {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first skipped","expected":"skip"}, + {"id":"serve-b","feature":"Serving","scenario":"serve-02 - second skipped","expected":"skip"} + ] + }"#; + let (_d, path) = write_platform(&report, platform); + let md = + consolidated_summary_markdown(&load_platform_reports(&[("mock".to_string(), path)])); + + for (id, name) in [ + ("serve-a", "serve-01 - first skipped"), + ("serve-b", "serve-02 - second skipped"), + ] { + assert!( + md.contains(&format!("{name}<br>[`{id}`](#{id})")), + "grid link should use the manifest name for {id}:\n{md}" + ); + assert!( + md.contains(&format!("##### {id}")), + "grid link for {id} should have a Scenario reference anchor:\n{md}" + ); + } + assert_eq!( + md.matches("_Not run on any platform in this run._").count(), + 2, + "each all-skipped reference entry should explain that it was not run:\n{md}" + ); + } + + #[test] + fn grid_reconciles_xfail_and_pass_by_id() { + // Scenario s0 tagged @id:serve-x, expected xfail, actually failed → xfail (good). + // Scenario s1 tagged @id:examine-y, expected pass, actually passed → pass. + let report = feature_json(&[ + (&["id:serve-x"], &["failed"]), + (&["id:examine-y"], &["passed"]), + ]); + let platform = r#"{ + "platform_slug": "mi300x", + "capability": {"effective_serve_engine": "vllm"}, + "expectations": [ + {"id":"serve-x","effective_engine":"vllm","expected":"xfail","bug":"EAI-7333","reason":"readiness gap"}, + {"id":"examine-y","effective_engine":"vllm","expected":"pass"} + ] + }"#; + let (_d, path) = write_platform(&report, platform); + let inputs = vec![("mi300x".to_string(), path)]; + let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); + assert!(md.contains("### Expectation grid"), "grid missing:\n{md}"); + assert!( + md.contains("[`serve-x`](#serve-x) | ✗ |"), + "serve-x should be xfail (grey ✗):\n{md}" + ); + assert!( + md.contains("[`examine-y`](#examine-y) | ✅ |"), + "examine-y should be pass:\n{md}" + ); + // No problems → no needs-attention from the grid. + assert!(!md.contains("**XPASS**"), "should have no XPASS:\n{md}"); + } + + #[test] + fn grid_flags_xpass_when_known_bug_passes() { + // s0 expected xfail but PASSED → XPASS (the run #543 Strix-Windows case). + let report = feature_json(&[(&["id:serve-default"], &["passed"])]); + let platform = r#"{ + "platform_slug": "strix-halo", + "capability": {"effective_serve_engine": "lemonade"}, + "expectations": [ + {"id":"serve-default","effective_engine":"lemonade","expected":"xfail","bug":"EAI-7333","reason":"readiness gap"} + ] + }"#; + let (_d, path) = write_platform(&report, platform); + let inputs = vec![("strix-halo".to_string(), path)]; + let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); + assert!(md.contains("⚠️XPASS"), "grid cell should show XPASS:\n{md}"); + // Canonical label ("Strix Halo Linux", via parse_descriptor's fallback + // titlecasing — fallback_descriptor always assigns Linux), not the raw + // "strix-halo" slug, and WITH the OS (the grid has no separate OS + // column, unlike the summary matrix) — see issue #702. + assert!( + md.contains("**XPASS** on `Strix Halo Linux`: `serve-default` (EAI-7333)"), + "needs-attention should list the XPASS with bug:\n{md}" + ); + } + + #[test] + fn grid_tolerates_flaky_xpass() { + let report = feature_json(&[(&["id:serve-flaky"], &["passed"])]); + let platform = r#"{ + "platform_slug": "mi300x", + "capability": {"effective_serve_engine": "vllm"}, + "expectations": [ + {"id":"serve-flaky","effective_engine":"vllm","expected":"xfail","bug":"EAI-7333","reason":"readiness race","flaky":true} + ] + }"#; + let (_d, path) = write_platform(&report, platform); + let inputs = vec![("mi300x".to_string(), path)]; + let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); + assert!( + md.contains("✅XPASS (flaky)"), + "grid should show tolerated flaky XPASS:\n{md}" + ); + assert!( + !md.contains("**XPASS** on"), + "flaky XPASS must not need attention:\n{md}" + ); + } + + #[test] + fn grid_heading_shows_component_versions() { + // platform.json carries OS/ROCm/vLLM/lemonade versions → they render in the + // summary matrix Platform cell (ROCm/vLLM/lemonade) and OS cell (distro). + // (Versions were removed from the expectation-grid header.) + let report = feature_json(&[(&["id:serve-x"], &["passed"])]); + let platform = r#"{ + "platform_slug": "mi300x", + "capability": {"effective_serve_engine": "vllm"}, + "versions": {"os":"Ubuntu 24.04.3 LTS","rocm":"7.13.0","vllm":"0.23.0+rocm723","lemonade":"11.5.1"}, + "expectations": [ + {"id":"serve-x","effective_engine":"vllm","expected":"pass"} + ] + }"#; + let (_d, path) = write_platform(&report, platform); + let inputs = vec![("mi300x".to_string(), path)]; + let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); + for token in [ + "Ubuntu 24.04.3 LTS", + "ROCm 7.13.0", + "vLLM 0.23.0+rocm723", + "lemonade 11.5.1", + ] { + assert!(md.contains(token), "matrix cell missing {token:?}:\n{md}"); + } + } + + #[test] + fn grid_heading_omits_absent_versions() { + // A platform.json with only OS known renders just the distro (in the matrix + // OS cell); absent ROCm/vLLM/lemonade are omitted from the Platform cell. + let report = feature_json(&[(&["id:serve-x"], &["passed"])]); + let platform = r#"{ + "platform_slug": "mock", + "capability": {"effective_serve_engine": "lemonade"}, + "versions": {"os":"Debian 12"}, + "expectations": [ + {"id":"serve-x","effective_engine":"lemonade","expected":"pass"} + ] + }"#; + let (_d, path) = write_platform(&report, platform); + let inputs = vec![("mock".to_string(), path)]; + let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); + assert!(md.contains("Debian 12"), "should show OS distro:\n{md}"); + assert!(!md.contains("vLLM "), "no vLLM version known → omit:\n{md}"); + } + + #[test] + fn grid_shows_skip_as_not_applicable() { + // Scenario is skip on this host; report.json has no entry for it. + let report = feature_json(&[(&["id:ran-here"], &["passed"])]); + let platform = r#"{ + "platform_slug": "mock", + "capability": {"effective_serve_engine": "lemonade"}, + "expectations": [ + {"id":"ran-here","effective_engine":"lemonade","expected":"pass"}, + {"id":"gpu-only","effective_engine":"vllm","expected":"skip","reason":"requires an AMD GPU"} + ] + }"#; + let (_d, path) = write_platform(&report, platform); + let inputs = vec![("mock".to_string(), path)]; + let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); + assert!( + md.contains("[`gpu-only`](#gpu-only) | n/a |"), + "skip should render as n/a:\n{md}" + ); + assert!(md.contains("[`ran-here`](#ran-here) | ✅ |")); + } +} diff --git a/crates/e2e-report/src/consolidated/mod.rs b/crates/e2e-report/src/consolidated/mod.rs index be1560a1af..5d357ca97a 100644 --- a/crates/e2e-report/src/consolidated/mod.rs +++ b/crates/e2e-report/src/consolidated/mod.rs @@ -3,24 +3,23 @@ // SPDX-License-Identifier: MIT //! Multi-platform consolidated report: the `PlatformReport` model, the -//! reconciled (scenario × platform) `Grid`, and the HTML/markdown generation -//! built on top of them. Artifact-name parsing lives in the `descriptor` -//! submodule and the `platform.json`/`commands.jsonl` schema and parsing in the -//! `sidecars` submodule. +//! reconciled (scenario × platform) `Grid`, and the `PlatformReports` bundle the +//! two renderers draw from. Artifact-name parsing lives in the `descriptor` +//! submodule, the `platform.json`/`commands.jsonl` schema and parsing in the +//! `sidecars` submodule, and the HTML and markdown renderers in the `html` and +//! `markdown` submodules. use std::collections::BTreeMap; use std::path::{Path, PathBuf}; -use maud::{Markup, html}; - -use crate::command_coverage::command_coverage_markdown; -use crate::components::{feature_group, now_utc, page_shell, stats_bar}; use crate::parse::{ EXPECTED_FAILURE_TAG, Feature, Stats, XfailReport, evaluate_xfail_features, parse_features, scenario_id, scenario_passed, stats_of, }; mod descriptor; +pub(super) mod html; +pub(super) mod markdown; mod sidecars; #[cfg(test)] mod test_support; @@ -31,40 +30,6 @@ use sidecars::{ parse_platform_manifest, }; -/// Run-level metadata shown in the report header so a downloaded report can be -/// traced back to the CI run that produced it. All optional — populated from CI -/// env vars, absent for a local run. -#[derive(Default)] -pub struct RunMeta { - pub commit: Option<String>, - pub branch: Option<String>, - pub run_number: Option<String>, - pub event: Option<String>, -} - -impl RunMeta { - fn line(&self) -> Option<String> { - let mut parts = Vec::new(); - if let Some(c) = &self.commit { - parts.push(format!("commit {}", &c[..c.len().min(7)])); - } - if let Some(b) = &self.branch { - parts.push(format!("branch {b}")); - } - if let Some(n) = &self.run_number { - parts.push(format!("run #{n}")); - } - if let Some(e) = &self.event { - parts.push(e.clone()); - } - if parts.is_empty() { - None - } else { - Some(parts.join(" · ")) - } - } -} - /// A single platform/job's parsed report plus its derived health. /// /// One of these corresponds to one uploaded `*-report` artifact (a @@ -771,8 +736,8 @@ fn report_sort_key(r: &PlatformReport) -> (&str, &str, bool, Option<&str>) { /// /// Holds the per-platform `PlatformReport`s, the reconciled expectation /// `Grid`, and the scenario-reference map. Returned by -/// [`load_platform_reports`] so [`generate_consolidated`] and -/// [`consolidated_summary_markdown`] draw every part they render from the +/// [`load_platform_reports`] so [`html::generate_consolidated`] and +/// [`markdown::consolidated_summary_markdown`] draw every part they render from the /// same platform set — the parts can no longer be independently filtered, /// reordered, or re-paired into a self-contradicting report the way loose /// parameters could (ROCMAI-484 / ROCm/rocm-cli#640). Opaque: nothing outside @@ -800,8 +765,8 @@ fn load_reports(inputs: &[(String, PathBuf)]) -> Vec<PlatformReport> { /// Parses every platform's `report.json`/`platform.json`/`commands.jsonl` /// into `PlatformReport`s (via [`load_reports`]), sorted by platform/OS, then /// known-bugs flag, then channel, and builds the expectation `Grid` and -/// scenario-reference map alongside them. Shared by [`generate_consolidated`] -/// and [`consolidated_summary_markdown`], so a caller producing both outputs +/// scenario-reference map alongside them. Shared by [`html::generate_consolidated`] +/// and [`markdown::consolidated_summary_markdown`], so a caller producing both outputs /// from the same `inputs` in one process (e.g. `xtask`'s `e2e_report::run`) /// builds each part once, not once per output or per report section. Each /// input's files are parsed exactly once, by `PlatformReport::load` @@ -832,471 +797,6 @@ fn platform_cell(r: &PlatformReport) -> String { with_channel_suffix(&r.desc.platform, r.effective_channel()) } -/// Build one consolidated HTML report from a [`PlatformReports`] bundle. -/// -/// The bundle is built from `(label, json_path)` pairs passed to -/// [`load_platform_reports`]; the label identifies the platform/tier (e.g. -/// "GPU Strix Windows (known bugs)"). New platforms need no code change — the -/// caller just passes more inputs there. -/// -/// Reads no input from disk; everything it renders comes from the bundle -/// (see [`load_platform_reports`]). -pub fn generate_consolidated( - bundle: &PlatformReports, - html_out: &Path, - meta: &RunMeta, -) -> std::io::Result<()> { - let reports = &bundle.reports; - let now = now_utc(); - let all_ok = reports.iter().all(PlatformReport::ok); - let overall = if reports.is_empty() { - ("status-fail", "No platform reports found".to_string()) - } else if all_ok { - ("status-pass", "All platforms in expected state".to_string()) - } else { - let bad = reports.iter().filter(|r| !r.ok()).count(); - ("status-fail", format!("{bad} platform(s) need attention")) - }; - - let body = html! { - div.header { - h1 { "Consolidated E2E Report" } - div.generated { - @if let Some(line) = meta.line() { (line) br; } - "Generated " (now) - } - } - - h2 { "Summary Information" } - table.summary-table { - tr { - td { "Status:" } - td class=(overall.0) { (overall.1) } - } - tr { td { "Rows:" } td { (reports.len()) } } - } - - @if reports.is_empty() { - p { "No per-platform report.json files were found to consolidate." } - } @else { - h2 { "Platforms" } - (matrix_table(reports)) - (legend()) - - (expectation_grid_html(&bundle.grid)) - - h2 { "Per-platform Details" } - div.details { - @for report in reports { - (platform_section(report)) - } - } - } - }; - let markup = page_shell("Consolidated E2E Report", body); - - std::fs::write(html_out, markup.into_string()) -} - -/// Render the consolidated result from a [`PlatformReports`] bundle as a -/// GitHub-flavoured markdown table, for piping into `$GITHUB_STEP_SUMMARY`. -/// Same bundle as [`generate_consolidated`]. -/// -/// Reads nothing from disk; everything it renders comes from the bundle -/// (see [`load_platform_reports`]). -pub fn consolidated_summary_markdown(bundle: &PlatformReports) -> String { - use std::fmt::Write as _; - - let reports = &bundle.reports; - let mut out = String::from("## E2E consolidated report\n\n"); - if reports.is_empty() { - out.push_str("_No per-platform report.json files were found to consolidate._\n"); - return out; - } - - out.push_str("| Platform | OS | Total | Pass | Fail | Skip | Xfail | Status |\n"); - out.push_str("|---|---|--:|--:|--:|--:|--:|:--|\n"); - let (mut t_total, mut t_pass, mut t_fail, mut t_skip, mut t_xfail) = (0, 0, 0, 0, 0); - for r in reports { - let (total, pass, fail, skip, xf) = r.display_counts(); - // Xfail column only applies where there are known bugs to invert; a plain - // expect-pass platform (no xfail entries) shows N/A. - let xfail = if xf > 0 || r.is_known_bugs { - xf.to_string() - } else { - "n/a".to_string() - }; - t_total += total; - t_pass += pass; - t_fail += fail; - t_skip += skip; - t_xfail += xf; - // Component versions in the Platform/OS cells: ROCm/vLLM/lemonade under the - // platform, the OS version under the OS. Absent components are omitted (mock - // has no runtime; a not-yet-probed source is simply skipped). - let plat_base = platform_cell(r); - let plat_cell = match r.versions.platform_stack() { - s if s.is_empty() => plat_base, - s => format!("{plat_base}<br><sub>{s}</sub>"), - }; - let os_cell = match r.versions.os.as_deref() { - Some(v) if v != r.desc.os => format!("{}<br><sub>{}</sub>", r.desc.os, v), - _ => r.desc.os.clone(), - }; - // `writeln!` into a String never fails; the discard keeps clippy happy. - let _ = writeln!( - out, - "| {} | {} | {} | {} | {} | {} | {} | {} |", - plat_cell, - os_cell, - total, - pass, - fail, - skip, - xfail, - r.status_text(), - ); - } - let _ = writeln!( - out, - "| **Total** | | {t_total} | {t_pass} | {t_fail} | {t_skip} | {t_xfail} | |", - ); - - out.push_str( - "\n**Mock** runs the real `rocm` CLI, but fakes the inference backend: instead \ - of downloading a runtime and launching a real engine (vLLM/lemonade) on a \ - GPU, a tiny in-process HTTP server stands in for the OpenAI-compatible \ - endpoint that engine would expose (`/v1/models`, `/v1/chat/completions`). \ - This exercises CLI behaviour — alias resolution, service discovery, chat \ - forwarding — with no GPU, no model download, and no engine process, so it \ - runs on a GitHub-hosted runner. It **gates the PR**: it runs on every push, \ - and if it fails the PR's required check goes red and the PR cannot merge. \ - **Every other platform** is a real self-hosted GPU host running real \ - engines — the rows below are the list, so naming them here would only go \ - stale as lanes are added. They are **non-blocking**: they still run and \ - are reported here, but \ - a failure does NOT block the PR from merging (the hardware/runners are still \ - being proven out, so their results are informational rather than a merge \ - gate).\n\n\ - Column meanings: **Pass** = scenarios that passed as expected; \ - **Xfail** = known bugs that failed as expected (healthy — the bug still \ - reproduces); **Skip** = not applicable on this platform (e.g. a GPU-only \ - scenario on Mock); **Fail** = unexpected — a scenario that should pass but \ - failed, or a known bug that unexpectedly passed (its tag is now stale). \ - **Status** is PASS unless a platform has any Fail.\n", - ); - - // Call out anything that needs a human: XPASS (fixed bug, stale tag) and - // untagged failures in a known-bugs run. - let mut notes = Vec::new(); - for r in reports { - for name in &r.xfail.xpass { - notes.push(format!( - "- **XPASS** in _{}_: `{}` is tagged `@expected-failure` but passed — remove the tag.", - r.label, name - )); - } - for name in &r.xfail.untagged_failures { - if r.is_known_bugs { - notes.push(format!( - "- **Regression** in _{}_: `{}` failed but is not tagged `@expected-failure`.", - r.label, name - )); - } - } - } - if !notes.is_empty() { - out.push_str("\n### Needs attention\n\n"); - for n in notes { - out.push_str(&n); - out.push('\n'); - } - } - - // The per-(scenario × platform) expectation grid, when platform.json - // sidecars are present (the new expectation system). Placed before the - // command-coverage table; it supersedes the coarse platform×tier matrix for - // "where should each test pass / not matter / not run". The scenario - // map supplies names for the grid's Scenario column (name + id link). - out.push_str(&expectation_grid_markdown(&bundle.grid, &bundle.scenarios)); - - out.push_str(&command_coverage_markdown(reports)); - - // Scenario reference LAST (after command coverage): each id's actual Gherkin - // scenario, anchored so the grid's id links resolve here. - out.push_str(&scenario_reference_markdown( - &bundle.grid, - &bundle.scenarios, - )); - - out -} - -/// Render the reconciled expectation grid as an HTML section (for the standalone -/// report). Empty markup only when there is truly nothing to report — no -/// column/row AND no excluded platform to caption (see -/// [`Grid::nothing_to_report`]); a report where every platform lacks a usable -/// `platform.json` still renders a heading and caption naming them. -fn expectation_grid_html(grid: &Grid) -> Markup { - // Not `grid.is_empty()`: a report where every platform lacks a - // `platform.json` has no columns/groups either, but still has an - // `excluded_note` worth showing — otherwise that case regresses to a - // silently missing section, the exact complaint issue #702 raises (just - // for every platform at once instead of some of them). - if grid.nothing_to_report() { - return html! {}; - } - let problems = grid.problems(); - let excluded_note = grid.excluded_note(); - html! { - h2 { "Expectation grid (scenario × platform)" } - @if !grid.is_empty() { - p.grid-legend { - @for (i, (outcome, text)) in LEGEND_ENTRIES.iter().copied().enumerate() { - @if i > 0 { " · " } - @if outcome.grid_class().is_empty() { - (outcome.glyph()) - } @else { - span class=(outcome.grid_class()) { (outcome.glyph()) } - } - " " (text) - } - } - } - @if let Some(note) = &excluded_note { - p.empty-note { (note) } - } - // One table per feature, so a reader can scan a single area of the CLI - // instead of one undivided 60-row block. - @for group in &grid.groups { - h3.feature-heading { (group.feature) } - table.stats { - thead { - tr { - th { "Scenario" } - @for col in &grid.columns { - @let versions = col.versions.summary(); - th { - (col.label()) - @if !col.engine.is_empty() { br; small { (col.engine) } } - @if !versions.is_empty() { br; small.versions { (versions) } } - } - } - } - } - tbody { - @for row in &group.rows { - tr { - td { - @if !row.name.is_empty() { (row.name) br; } - code { (row.id) } - } - @for col in &grid.columns { - @let outcome = col.outcomes.get(&row.id).copied().unwrap_or(CellOutcome::Missing); - // One combined class attr — `td.num class=(..)` would emit - // two `class` attributes, and the browser keeps only the - // first ("num"), dropping the status colour. - td class=(format!("num {}", outcome.grid_class())) { - (outcome.glyph()) - } - } - } - } - } - } - } - @if !problems.is_empty() { - h3 { "Needs attention" } - ul { - @for (platform_label, id, outcome, detail) in &problems { - li { - b { - @match outcome { - CellOutcome::Xpass => "XPASS", - CellOutcome::UnexpectedFail => "unexpected failure", - CellOutcome::RanWhenNa => "ran despite n/a", - _ => "issue", - } - } - " on " code { (platform_label) } ": " code { (id) } - @if let Some(d) = detail { - @if let Some(b) = &d.bug { " (" (b) ")" } - @if !d.effective_engine.is_empty() { " [engine: " (d.effective_engine) "]" } - @if let Some(r) = &d.reason { " — " (r) } - } - } - } - } - } - } -} - -/// Render the reconciled (scenario-id × platform) expectation grid as markdown, -/// plus a "needs attention" list of every XPASS / unexpected-fail / ran-when-NA. -/// Empty string only when there is truly nothing to report — no column/row -/// AND no excluded platform to caption (see [`Grid::nothing_to_report`]); a -/// report where every platform lacks a usable `platform.json` still renders a -/// heading and caption naming them. -fn expectation_grid_markdown(grid: &Grid, scenarios: &ScenarioReference) -> String { - use std::fmt::Write as _; - - // Not `grid.is_empty()`: see the matching comment in `expectation_grid_html` - // — a report where every platform lacks a `platform.json` still has an - // `excluded_note` worth showing, even with no columns/groups to table. - if grid.nothing_to_report() { - return String::new(); - } - - let mut out = String::from("\n### Expectation grid (scenario × platform)\n\n"); - if !grid.is_empty() { - let legend = LEGEND_ENTRIES - .iter() - .map(|(outcome, text)| format!("{} {text}", outcome.glyph())) - .collect::<Vec<_>>() - .join(" · "); - let _ = writeln!(out, "_{legend}_\n"); - } - if let Some(note) = grid.excluded_note() { - let _ = writeln!(out, "_{note}_\n"); - } - - // One table per feature, under its own heading — a single undivided table of - // every scenario in the suite is unreadable, and gives no clue where one area - // of the CLI ends and the next begins. - for group in &grid.groups { - let _ = writeln!(out, "#### {}\n", group.feature); - - // Header: one column per platform, with its effective engine. Component - // versions live in the summary matrix above, not here. - out.push_str("| Scenario |"); - for col in &grid.columns { - let eng = if col.engine.is_empty() { - String::new() - } else { - format!("<br><sub>{}</sub>", col.engine) - }; - let _ = write!(out, " {}{} |", col.label(), eng); - } - out.push('\n'); - out.push_str("|---|"); - for _ in &grid.columns { - out.push_str(":--:|"); - } - out.push('\n'); - - for row in &group.rows { - // Scenario cell: human name on top, the @id below as a link to its - // entry in the Scenario reference section (GitHub anchors - // `##### <id>` to `#<id>`). Prefer the name recorded in platform.json - // — it covers a scenario that was skipped everywhere and so has no - // report.json entry to read a name from. - let name = if row.name.is_empty() { - scenarios.get(&row.id).map_or("", |(n, _)| n.as_str()) - } else { - row.name.as_str() - }; - let id = &row.id; - let _ = write!(out, "| {name}<br>[`{id}`](#{id}) |"); - for col in &grid.columns { - let g = col - .outcomes - .get(id) - .copied() - .unwrap_or(CellOutcome::Missing); - let _ = write!(out, " {} |", g.glyph()); - } - out.push('\n'); - } - out.push('\n'); - } - - // Needs-attention list from reconciliation problems. - let problems = grid.problems(); - if !problems.is_empty() { - out.push_str("\n### Needs attention\n\n"); - for (platform_label, id, outcome, detail) in problems { - let bug = detail - .and_then(|d| d.bug.as_deref()) - .map(|b| format!(" ({b})")) - .unwrap_or_default(); - let engine = detail - .map(|d| d.effective_engine.as_str()) - .filter(|e| !e.is_empty()) - .map(|e| format!(" [engine: {e}]")) - .unwrap_or_default(); - let reason = detail - .and_then(|d| d.reason.as_deref()) - .map(|r| format!(" — {r}")) - .unwrap_or_default(); - let kind = match outcome { - CellOutcome::Xpass => "XPASS", - CellOutcome::UnexpectedFail => "unexpected failure", - CellOutcome::RanWhenNa => "ran despite n/a", - _ => "issue", - }; - let _ = writeln!( - out, - "- **{kind}** on `{platform_label}`: `{id}`{bug}{engine}{reason}" - ); - } - } - - out -} - -/// Render the Scenario reference section: each scenario `@id` with its actual -/// Gherkin scenario (name + steps), anchored by the id (`##### <id>` → GitHub -/// anchor `#<id>`) so the grid's id links resolve. Grouped by feature and ordered -/// like the grid. Empty when no expectation grid exists. -fn scenario_reference_markdown(grid: &Grid, scenarios: &ScenarioReference) -> String { - use std::fmt::Write as _; - - // Only emit when there is a grid to reference (platform.json sidecars present), - // matching where the id links are generated. - if grid.is_empty() { - return String::new(); - } - - // Walk the grid's own order so the reference is laid out feature by feature, - // matching the tables that link into it. Every grid row gets an entry — a - // scenario skipped on every platform has no steps to show, but still needs - // its anchor or the grid's link to it dangles. - // - // Scoped to grid rows deliberately: this section exists to back the grid's - // links, so it documents exactly what the grid shows. A scenario present only - // in an artifact with no `platform.json` sidecar has no grid row and so gets - // no entry — nothing links to it either. - let mut out = String::from("\n### Scenario reference\n\n"); - for group in &grid.groups { - let _ = writeln!(out, "#### {}\n", group.feature); - for row in &group.rows { - let entry = scenarios.get(&row.id); - // Prefer the platform.json name, as the grid does — it is the only - // source that covers a scenario which ran nowhere. - let name = if row.name.is_empty() { - entry.map_or("", |(n, _)| n.as_str()) - } else { - row.name.as_str() - }; - let _ = writeln!(out, "##### {}\n", row.id); - if !name.is_empty() { - let _ = writeln!(out, "_{name}_\n"); - } - match entry { - Some((_, steps)) => { - for step in steps { - let _ = writeln!(out, "- {step}"); - } - } - // Ran nowhere (n/a on every platform), so no steps were recorded. - None => out.push_str("_Not run on any platform in this run._\n"), - } - out.push('\n'); - } - } - out -} - /// Scenario `@id` → (human name, Gherkin steps); see [`scenario_reference`]. type ScenarioReference = BTreeMap<String, (String, Vec<String>)>; @@ -1325,110 +825,10 @@ fn scenario_reference(reports: &[PlatformReport]) -> ScenarioReference { map } -fn matrix_table(reports: &[PlatformReport]) -> Markup { - let (mut t_total, mut t_pass, mut t_fail, mut t_skip, mut t_xfail) = (0, 0, 0, 0, 0); - for r in reports { - let (total, pass, fail, skip, xf) = r.display_counts(); - t_total += total; - t_pass += pass; - t_fail += fail; - t_skip += skip; - t_xfail += xf; - } - html! { - table.stats { - tr { - th { "Platform" } th { "OS" } - th { "Total" } th { "Pass" } th { "Fail" } th { "Skip" } - th { "Xfail" } th { "Status" } - th { "Pass / Fail / Skip" } - } - @for r in reports { - @let (total, pass, fail, skip, xf) = r.display_counts(); - tr { - td { (platform_cell(r)) } - td { (r.desc.os) } - td.num { (total) } - td.num { (pass) } - td.num { (fail) } - td.num { (skip) } - td.num { @if xf > 0 || r.is_known_bugs { (xf) } @else { "n/a" } } - td class=(if r.ok() { "status-pass" } else { "status-fail" }) { - (r.status_text()) - } - td { (stats_bar(&r.stats)) } - } - } - tr.total-row { - td { "Total" } td {} - td.num { (t_total) } - td.num { (t_pass) } - td.num { (t_fail) } - td.num { (t_skip) } - td.num { (t_xfail) } - td {} td {} - } - } - } -} - -/// Explain the non-obvious columns/terms so the report is self-contained. -fn legend() -> Markup { - html! { - div.legend { - h3 { "Legend" } - ul { - li { - b { "Mock" } - " — no GPU. The CLI runs against a fake in-process model " - "server (a planted service record), validating CLI behaviour " - "and wiring without hardware. Runs on a GitHub-hosted runner " - "and gates the PR." - } - li { - b { "Every other platform" } - " — real self-hosted GPU hardware; non-blocking while proven out." - } - li { - b { "Status" } - " — PASS when the platform is in its expected state (every " - "expect-pass scenario passed and every known bug failed as " - "expected); FAIL on an unexpected failure or an XPASS (a known " - "bug that unexpectedly passed — remove its @expected-failure tag)." - } - li { b { "Xfail" } " — count of known-bug scenarios that failed as expected." } - li { b { "Skip" } " — scenarios not applicable on this platform (not run)." } - } - } - } -} - -fn platform_section(report: &PlatformReport) -> Markup { - let badge_class = if report.ok() { - "badge-pass" - } else { - "badge-fail" - }; - html! { - details.platform open[!report.ok()] { - summary.platform-row { - span class=(format!("badge {badge_class}")) { (report.status_text()) } - span.platform-name { (report.label) } - span.elapsed { (report.stats.total) " scenarios" } - } - @if report.features.is_empty() { - p.empty-note { "No report.json data for this platform." } - } @else { - @for feature in &report.features { - (feature_group(feature)) - } - } - } - } -} - #[cfg(test)] mod tests { + use super::html::{RunMeta, expectation_grid_html, generate_consolidated}; + use super::markdown::consolidated_summary_markdown; use super::test_support::{feature_json, grid_order, write_platform, write_report}; use super::*; use crate::command_coverage::{KNOWN_COMMAND_SURFACE, command_coverage_summary}; @@ -1479,72 +879,6 @@ mod tests { assert_eq!(r.status_text(), "EMPTY"); } - #[test] - fn consolidated_summary_markdown_has_a_row_per_platform() { - let a = write_report(&feature_json(&[(&[], &["passed"]), (&[], &["passed"])])); - let b = write_report(&feature_json(&[(&["expected-failure"], &["failed"])])); - let inputs = vec![ - ("e2e-report".to_string(), a.path().to_path_buf()), - ( - "e2e-gpu-known-bugs-report".to_string(), - b.path().to_path_buf(), - ), - ]; - let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - assert!(md.contains("| Mock | Linux | 2 | 2 | 0 | 0 | n/a | PASS |")); - assert!(md.contains("| MI300X | Linux | 1 | 0 | 1 | 0 | 1 | PASS |")); - } - - #[test] - fn consolidated_summary_markdown_flags_xpass() { - let b = write_report(&feature_json(&[(&["expected-failure"], &["passed"])])); - let inputs = vec![( - "e2e-gpu-known-bugs-report".to_string(), - b.path().to_path_buf(), - )]; - let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - assert!(md.contains("Needs attention")); - assert!(md.contains("XPASS")); - } - - #[test] - fn consolidated_summary_markdown_empty_inputs() { - let md = consolidated_summary_markdown(&load_platform_reports(&[])); - assert!(md.contains("No per-platform report.json files")); - } - - #[test] - fn generate_consolidated_renders_the_expectation_grid() { - // Review finding (6f130efe round, back when `generate_consolidated` - // was `generate_consolidated_from_reports` and took `reports` and - // `inputs` as two loose parameters): passing `&[]` instead of `inputs` - // to the `expectation_grid_html(inputs)` call inside passed every - // existing test at the time, because the fixtures used elsewhere had - // no platform.json, so the grid was empty either way. The same slip - // now would be building the bundle's grid from `&[]` in - // `load_platform_reports`. Give this one a platform.json via - // write_platform and assert the grid heading actually renders. - let report = feature_json(&[(&["id:serve-a"], &["passed"])]); - let platform = r#"{ - "platform_slug": "mock", - "capability": {"effective_serve_engine": "none"}, - "expectations": [ - {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"pass"} - ] - }"#; - let (_d, path) = write_platform(&report, platform); - let inputs = vec![("mock".to_string(), path)]; - let bundle = load_platform_reports(&inputs); - - let out = tempfile::NamedTempFile::new().expect("temp"); - generate_consolidated(&bundle, out.path(), &RunMeta::default()).expect("generate"); - let html = std::fs::read_to_string(out.path()).expect("read"); - assert!( - html.contains("Expectation grid (scenario × platform)"), - "grid heading missing from output:\n{html}" - ); - } - #[test] fn rendering_uses_the_bundle_not_the_artifacts_on_disk() { // The render functions must work from what `load_platform_reports` @@ -1882,62 +1216,6 @@ mod tests { ); } - #[test] - fn generate_consolidated_writes_html() { - // Same platform_slug, two channels: the HTML matrix (matrix_table) must - // carry the channel suffix too, not just its markdown twin. - let release_platform = r#"{ - "platform_slug": "mi300x", - "capability": {"effective_serve_engine": "vllm"}, - "versions": {"channel": "release"} - }"#; - let nightly_platform = r#"{ - "platform_slug": "mi300x", - "capability": {"effective_serve_engine": "vllm"}, - "versions": {"channel": "nightly"} - }"#; - let report = feature_json(&[(&[], &["passed"])]); - let (_d1, release_path) = write_platform(&report, release_platform); - let (_d2, nightly_path) = write_platform(&report, nightly_platform); - let out = tempfile::NamedTempFile::new().expect("temp"); - let inputs = vec![ - ("mi300x-release".to_string(), release_path), - ("mi300x-nightly".to_string(), nightly_path), - ]; - generate_consolidated( - &load_platform_reports(&inputs), - out.path(), - &RunMeta::default(), - ) - .expect("generate"); - let html = std::fs::read_to_string(out.path()).expect("read"); - assert!(html.contains("Consolidated E2E Report")); - assert!( - html.contains("<title>Consolidated E2E Report"), - "the page_shell title argument must reach the tag, not just the <h1>" - ); - assert!(html.contains("Platforms")); - assert!(html.contains("Legend")); - assert!( - html.contains("Mi300x (release)"), - "HTML matrix must carry the release channel suffix" - ); - assert!( - html.contains("Mi300x (nightly)"), - "HTML matrix must carry the nightly channel suffix" - ); - - // The sort key's channel element only matters if it actually drives - // order: nightly's lexical head start ("n" < "r") must place it before - // release in the rendered matrix, or this element of the tuple is dead. - let nightly_pos = html.find("Mi300x (nightly)").expect("nightly cell present"); - let release_pos = html.find("Mi300x (release)").expect("release cell present"); - assert!( - nightly_pos < release_pos, - "HTML matrix must order nightly before release for the same platform" - ); - } - #[test] fn command_coverage_ties_to_scenario_not_rc() { // A scenario that PASSES while its command exited non-zero (e.g. an @@ -2023,58 +1301,6 @@ mod tests { assert_eq!(scenario_index(""), None); } - #[test] - fn grid_groups_by_feature_and_orders_by_index() { - // Declared deliberately out of order (and interleaved across features) - // so a passing assertion can only come from real grouping + sorting, - // not from the input order. - let report = feature_json(&[ - (&["id:serve-b"], &["passed"]), - (&["id:examine-a"], &["passed"]), - ]); - let platform = r#"{ - "platform_slug": "mock", - "capability": {"effective_serve_engine": "none"}, - "expectations": [ - {"id":"serve-b","feature":"Serving","scenario":"serve-02 - second","expected":"pass"}, - {"id":"examine-a","feature":"Examine","scenario":"examine-09 - ninth","expected":"pass"}, - {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first","expected":"skip"}, - {"id":"examine-b","feature":"Examine","scenario":"examine-10 - tenth","expected":"skip"} - ] - }"#; - let (_d, path) = write_platform(&report, platform); - let inputs = vec![("mock".to_string(), path)]; - - assert_eq!( - grid_order(&inputs), - vec![ - ("Examine".to_string(), "examine-a".to_string()), - ("Examine".to_string(), "examine-b".to_string()), - ("Serving".to_string(), "serve-a".to_string()), - ("Serving".to_string(), "serve-b".to_string()), - ], - "rows must group by feature and sort by index (09 before 10, not \ - lexically), regardless of declaration order", - ); - - // Each feature gets its own table under its own heading. - let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - assert!( - md.contains("#### Examine"), - "missing feature heading:\n{md}" - ); - assert!( - md.contains("#### Serving"), - "missing feature heading:\n{md}" - ); - // The human name is shown alongside the id, sourced from platform.json — - // serve-a was skipped everywhere, so report.json has no name for it. - assert!( - md.contains("serve-01 - first<br>[`serve-a`](#serve-a)"), - "skipped scenario should still show its name:\n{md}" - ); - } - #[test] fn grid_falls_back_to_report_feature_then_id_prefix() { // A platform.json predating the `feature`/`scenario` fields. `serve-x` @@ -2319,163 +1545,6 @@ mod tests { ); } - #[test] - fn consolidated_summary_markdown_orders_rows_by_report_sort_key() { - // report_sort_key_orders_by_every_dimension already pins all four - // tuple elements via a direct call to report_sort_key with synthetic - // PlatformReports. This test instead goes through the public - // consolidated_summary_markdown entry point with realistic - // artifact-name fixtures, so a regression in load_platform_reports' - // use of report_sort_key — e.g. calling it with the wrong field order, - // or not calling it at all — would still be caught even if - // report_sort_key itself stayed correct in isolation. - // - // Pins all four elements of the sort-key tuple, in precedence order: - // platform, os, known_bugs, channel. Three Strix Halo rows alone - // can't pin `platform` — they all share it, so dropping or swapping - // that element would not change their order — so a fourth row on a - // different platform ("Unknown"/"Unknown", via the explicit - // "e2e-unknown-report" artifact name) is added: "Strix Halo" sorts - // before "Unknown" by platform, but "Unknown" alone (as an OS) sorts - // before "Windows", so dropping the platform element from the key, or - // swapping it with os, both move this row ahead of the Windows row - // instead of after it. Within the Strix Halo rows: "Strix Halo"/Ubuntu - // before "Strix Halo"/Windows pins os; the two Ubuntu rows, identical - // except for the known-bugs artifact suffix, pin known_bugs ordering - // (false < true) once os and channel are held equal. A fifth and - // sixth row on two more distinct platforms with opposite channels — - // MI300X/release and R9700/nightly — pin channel's precedence below - // platform: "nightly" sorts before "release", so if channel outranked - // platform in the key, R9700 would sort first; "MI300X" sorts before - // "R9700" by platform, so the correct key keeps MI300X first - // regardless. A seventh row, same platform/os as the MI300X row but - // known_bugs=true and channel=nightly, pins that known_bugs outranks - // channel too: "nightly" sorts before "release", so if channel - // outranked known_bugs, this row would sort before the MI300X/release - // row despite known_bugs=false < true. Channel's own effect on - // ordering — as a tie-breaker once platform/os/known_bugs are equal — - // is pinned by generate_consolidated_writes_html and - // command_coverage_distinguishes_channels_on_same_platform, not - // repeated here. - // No platform.json sidecar, so display_counts falls back to raw junit - // stats and each row's distinct scenario count (1/2/3/4/5/6/7) tells - // the rows apart in the rendered table. - let ubuntu_plain_report = write_report(&feature_json(&[(&[], &["passed"])])); - let ubuntu_known_bugs_report = - write_report(&feature_json(&[(&[], &["passed"]), (&[], &["passed"])])); - let windows_plain_report = write_report(&feature_json(&[ - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - ])); - let unknown_report = write_report(&feature_json(&[ - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - ])); - let mi300x_release_report = write_report(&feature_json(&[ - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - ])); - let r9700_nightly_report = write_report(&feature_json(&[ - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - ])); - let mi300x_known_bugs_nightly_report = write_report(&feature_json(&[ - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - (&[], &["passed"]), - ])); - let inputs = vec![ - ( - "e2e-gpu-strix-windows-report".to_string(), - windows_plain_report.path().to_path_buf(), - ), - ( - "e2e-gpu-strix-ubuntu-known-bugs-report".to_string(), - ubuntu_known_bugs_report.path().to_path_buf(), - ), - ( - "e2e-gpu-strix-ubuntu-report".to_string(), - ubuntu_plain_report.path().to_path_buf(), - ), - ( - "e2e-unknown-report".to_string(), - unknown_report.path().to_path_buf(), - ), - ( - "e2e-gpu-release-report".to_string(), - mi300x_release_report.path().to_path_buf(), - ), - ( - "e2e-gpu-rad3-nightly-report".to_string(), - r9700_nightly_report.path().to_path_buf(), - ), - ( - "e2e-gpu-known-bugs-nightly-report".to_string(), - mi300x_known_bugs_nightly_report.path().to_path_buf(), - ), - ]; - - let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - let ubuntu_plain_pos = md - .find("| Strix Halo | Ubuntu | 1 |") - .expect("plain Ubuntu row"); - let ubuntu_known_bugs_pos = md - .find("| Strix Halo | Ubuntu | 2 |") - .expect("known-bugs Ubuntu row"); - let windows_pos = md - .find("| Strix Halo | Windows | 3 |") - .expect("Windows row"); - let unknown_pos = md - .find("| Unknown | Unknown | 4 |") - .expect("Unknown-platform row"); - let mi300x_pos = md - .find("| MI300X (release) | Linux | 5 |") - .expect("MI300X release row"); - let r9700_pos = md - .find("| R9700 (nightly) | Linux | 6 |") - .expect("R9700 nightly row"); - let mi300x_known_bugs_nightly_pos = md - .find("| MI300X (nightly) | Linux | 7 |") - .expect("MI300X known-bugs nightly row"); - assert!( - ubuntu_plain_pos < ubuntu_known_bugs_pos, - "same platform/os: known_bugs=false must sort before known_bugs=true:\n{md}" - ); - assert!( - ubuntu_known_bugs_pos < windows_pos, - "same platform: Ubuntu must sort before Windows regardless of known_bugs:\n{md}" - ); - assert!( - windows_pos < unknown_pos, - "\"Strix Halo\" must sort before \"Unknown\" by platform, even though \"Unknown\" as an OS sorts before \"Windows\":\n{md}" - ); - assert!( - mi300x_pos < r9700_pos, - "\"MI300X\" must sort before \"R9700\" by platform, even though its channel \ - (\"release\") sorts after R9700's (\"nightly\"):\n{md}" - ); - assert!( - mi300x_pos < mi300x_known_bugs_nightly_pos, - "same platform/os: known_bugs=false must sort before known_bugs=true, even though \ - the known_bugs=true row's channel (\"nightly\") sorts before the known_bugs=false \ - row's (\"release\"):\n{md}" - ); - } - #[test] fn nightly_failure_reaches_needs_attention_in_its_own_column() { // Pins that a release pass and a nightly fail on the same slug land in @@ -2586,148 +1655,6 @@ mod tests { assert_eq!(grid.columns[0].label(), "Mock Linux"); } - #[test] - fn scenario_reference_anchors_every_grid_row() { - // A scenario that is n/a everywhere has no report.json entry — it still - // needs an anchor, or the grid's link to it dangles. - let report = feature_json(&[(&["id:serve-x"], &["passed"])]); - let platform = r#"{ - "platform_slug": "mock", - "capability": {"effective_serve_engine": "none"}, - "expectations": [ - {"id":"serve-x","feature":"Serving","scenario":"serve-01 - ran","expected":"pass"}, - {"id":"serve-z","feature":"Serving","scenario":"serve-02 - skipped","expected":"skip"} - ] - }"#; - let (_d, path) = write_platform(&report, platform); - let md = - consolidated_summary_markdown(&load_platform_reports(&[("mock".to_string(), path)])); - assert!(md.contains("##### serve-x"), "missing anchor:\n{md}"); - assert!( - md.contains("##### serve-z"), - "a never-run scenario still needs its anchor:\n{md}" - ); - assert!( - md.contains("_Not run on any platform in this run._"), - "a never-run scenario should say so instead of showing no steps:\n{md}" - ); - } - - #[test] - fn scenario_reference_anchors_grid_when_report_has_no_scenarios() { - let report = feature_json(&[]); - let platform = r#"{ - "platform_slug": "mock", - "capability": {"effective_serve_engine": "none"}, - "expectations": [ - {"id":"serve-a","feature":"Serving","scenario":"serve-01 - first skipped","expected":"skip"}, - {"id":"serve-b","feature":"Serving","scenario":"serve-02 - second skipped","expected":"skip"} - ] - }"#; - let (_d, path) = write_platform(&report, platform); - let md = - consolidated_summary_markdown(&load_platform_reports(&[("mock".to_string(), path)])); - - for (id, name) in [ - ("serve-a", "serve-01 - first skipped"), - ("serve-b", "serve-02 - second skipped"), - ] { - assert!( - md.contains(&format!("{name}<br>[`{id}`](#{id})")), - "grid link should use the manifest name for {id}:\n{md}" - ); - assert!( - md.contains(&format!("##### {id}")), - "grid link for {id} should have a Scenario reference anchor:\n{md}" - ); - } - assert_eq!( - md.matches("_Not run on any platform in this run._").count(), - 2, - "each all-skipped reference entry should explain that it was not run:\n{md}" - ); - } - - #[test] - fn grid_reconciles_xfail_and_pass_by_id() { - // Scenario s0 tagged @id:serve-x, expected xfail, actually failed → xfail (good). - // Scenario s1 tagged @id:examine-y, expected pass, actually passed → pass. - let report = feature_json(&[ - (&["id:serve-x"], &["failed"]), - (&["id:examine-y"], &["passed"]), - ]); - let platform = r#"{ - "platform_slug": "mi300x", - "capability": {"effective_serve_engine": "vllm"}, - "expectations": [ - {"id":"serve-x","effective_engine":"vllm","expected":"xfail","bug":"EAI-7333","reason":"readiness gap"}, - {"id":"examine-y","effective_engine":"vllm","expected":"pass"} - ] - }"#; - let (_d, path) = write_platform(&report, platform); - let inputs = vec![("mi300x".to_string(), path)]; - let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - assert!(md.contains("### Expectation grid"), "grid missing:\n{md}"); - assert!( - md.contains("[`serve-x`](#serve-x) | ✗ |"), - "serve-x should be xfail (grey ✗):\n{md}" - ); - assert!( - md.contains("[`examine-y`](#examine-y) | ✅ |"), - "examine-y should be pass:\n{md}" - ); - // No problems → no needs-attention from the grid. - assert!(!md.contains("**XPASS**"), "should have no XPASS:\n{md}"); - } - - #[test] - fn grid_flags_xpass_when_known_bug_passes() { - // s0 expected xfail but PASSED → XPASS (the run #543 Strix-Windows case). - let report = feature_json(&[(&["id:serve-default"], &["passed"])]); - let platform = r#"{ - "platform_slug": "strix-halo", - "capability": {"effective_serve_engine": "lemonade"}, - "expectations": [ - {"id":"serve-default","effective_engine":"lemonade","expected":"xfail","bug":"EAI-7333","reason":"readiness gap"} - ] - }"#; - let (_d, path) = write_platform(&report, platform); - let inputs = vec![("strix-halo".to_string(), path)]; - let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - assert!(md.contains("⚠️XPASS"), "grid cell should show XPASS:\n{md}"); - // Canonical label ("Strix Halo Linux", via parse_descriptor's fallback - // titlecasing — fallback_descriptor always assigns Linux), not the raw - // "strix-halo" slug, and WITH the OS (the grid has no separate OS - // column, unlike the summary matrix) — see issue #702. - assert!( - md.contains("**XPASS** on `Strix Halo Linux`: `serve-default` (EAI-7333)"), - "needs-attention should list the XPASS with bug:\n{md}" - ); - } - - #[test] - fn grid_tolerates_flaky_xpass() { - let report = feature_json(&[(&["id:serve-flaky"], &["passed"])]); - let platform = r#"{ - "platform_slug": "mi300x", - "capability": {"effective_serve_engine": "vllm"}, - "expectations": [ - {"id":"serve-flaky","effective_engine":"vllm","expected":"xfail","bug":"EAI-7333","reason":"readiness race","flaky":true} - ] - }"#; - let (_d, path) = write_platform(&report, platform); - let inputs = vec![("mi300x".to_string(), path)]; - let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - assert!( - md.contains("✅XPASS (flaky)"), - "grid should show tolerated flaky XPASS:\n{md}" - ); - assert!( - !md.contains("**XPASS** on"), - "flaky XPASS must not need attention:\n{md}" - ); - } - #[test] fn expectation_grid_legend_explains_every_glyph() { // The legend must explain every glyph `CellOutcome::glyph()` can emit, @@ -2811,113 +1738,6 @@ mod tests { } } - #[test] - fn expectation_grid_html_legend_colors_every_red_glyph() { - // Every outcome that renders red in the grid (`grid_class() == - // "status-fail"`) must also render red in the legend, not just the - // original Xfail/UnexpectedFail pair — Xpass, RanWhenNa, and Absent - // are equally `status-fail` in the grid and were previously left as - // plain, uncoloured glyphs in the legend. - let report = feature_json(&[(&["id:serve-x"], &["passed"])]); - let platform = r#"{ - "platform_slug": "mi300x", - "capability": {"effective_serve_engine": "vllm"}, - "expectations": [ - {"id":"serve-x","effective_engine":"vllm","expected":"pass"} - ] - }"#; - let (_d, path) = write_platform(&report, platform); - let inputs = vec![("mi300x".to_string(), path)]; - - let html = expectation_grid_html(&Grid::build(&load_reports(&inputs))).into_string(); - for glyph in ["⚠️XPASS", "⚠️n/a-ran", "⚠️no-result", "❌FAIL"] { - let needle = format!("class=\"status-fail\">{glyph}</span>"); - assert!( - html.contains(&needle), - "legend must colour {glyph:?} status-fail:\n{html}" - ); - } - // The known-bug xfail glyph is styled grey (status-xfail), not red. - assert!( - html.contains("class=\"status-xfail\">✗</span>"), - "legend must colour the xfail glyph status-xfail:\n{html}" - ); - // Genuinely unstyled outcomes stay bare glyphs, not empty-class spans. - assert!( - !html.contains("class=\"\">"), - "unstyled legend glyphs must not be wrapped in an empty-class span:\n{html}" - ); - } - - #[test] - fn grid_heading_shows_component_versions() { - // platform.json carries OS/ROCm/vLLM/lemonade versions → they render in the - // summary matrix Platform cell (ROCm/vLLM/lemonade) and OS cell (distro). - // (Versions were removed from the expectation-grid header.) - let report = feature_json(&[(&["id:serve-x"], &["passed"])]); - let platform = r#"{ - "platform_slug": "mi300x", - "capability": {"effective_serve_engine": "vllm"}, - "versions": {"os":"Ubuntu 24.04.3 LTS","rocm":"7.13.0","vllm":"0.23.0+rocm723","lemonade":"11.5.1"}, - "expectations": [ - {"id":"serve-x","effective_engine":"vllm","expected":"pass"} - ] - }"#; - let (_d, path) = write_platform(&report, platform); - let inputs = vec![("mi300x".to_string(), path)]; - let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - for token in [ - "Ubuntu 24.04.3 LTS", - "ROCm 7.13.0", - "vLLM 0.23.0+rocm723", - "lemonade 11.5.1", - ] { - assert!(md.contains(token), "matrix cell missing {token:?}:\n{md}"); - } - } - - #[test] - fn grid_heading_omits_absent_versions() { - // A platform.json with only OS known renders just the distro (in the matrix - // OS cell); absent ROCm/vLLM/lemonade are omitted from the Platform cell. - let report = feature_json(&[(&["id:serve-x"], &["passed"])]); - let platform = r#"{ - "platform_slug": "mock", - "capability": {"effective_serve_engine": "lemonade"}, - "versions": {"os":"Debian 12"}, - "expectations": [ - {"id":"serve-x","effective_engine":"lemonade","expected":"pass"} - ] - }"#; - let (_d, path) = write_platform(&report, platform); - let inputs = vec![("mock".to_string(), path)]; - let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - assert!(md.contains("Debian 12"), "should show OS distro:\n{md}"); - assert!(!md.contains("vLLM "), "no vLLM version known → omit:\n{md}"); - } - - #[test] - fn grid_shows_skip_as_not_applicable() { - // Scenario is skip on this host; report.json has no entry for it. - let report = feature_json(&[(&["id:ran-here"], &["passed"])]); - let platform = r#"{ - "platform_slug": "mock", - "capability": {"effective_serve_engine": "lemonade"}, - "expectations": [ - {"id":"ran-here","effective_engine":"lemonade","expected":"pass"}, - {"id":"gpu-only","effective_engine":"vllm","expected":"skip","reason":"requires an AMD GPU"} - ] - }"#; - let (_d, path) = write_platform(&report, platform); - let inputs = vec![("mock".to_string(), path)]; - let md = consolidated_summary_markdown(&load_platform_reports(&inputs)); - assert!( - md.contains("[`gpu-only`](#gpu-only) | n/a |"), - "skip should render as n/a:\n{md}" - ); - assert!(md.contains("[`ran-here`](#ran-here) | ✅ |")); - } - // Pure helper unit tests live with their code in `command_coverage.rs`; // the command-coverage tests here stay because they build `PlatformReport` // through the private `PlatformReport::load` and the shared `feature_json` diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index ab70dfdb58..90007e194f 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -14,9 +14,8 @@ mod consolidated; mod parse; mod single_report; -pub use consolidated::{ - PlatformReports, RunMeta, consolidated_summary_markdown, generate_consolidated, - load_platform_reports, -}; +pub use consolidated::html::{RunMeta, generate_consolidated}; +pub use consolidated::markdown::consolidated_summary_markdown; +pub use consolidated::{PlatformReports, load_platform_reports}; pub use parse::{XfailReport, evaluate_xfail, scenario_results_by_id}; pub use single_report::generate; diff --git a/docs/architecture.md b/docs/architecture.md index ecb4709327..a1dc0aa28e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -206,14 +206,16 @@ Modularized (ROCMAI-60, Phase 1 of ROCMAI-27's sequencing): - [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) — (ROCMAI-486, Phase 1 of a staged further split of [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs)) the CLI command-coverage audit table — which `rocm` commands were exercised and whether they passed — consumes `consolidated::PlatformReport` but shares no type with the `Grid`/reconciliation model itself. - [`components.rs`](../crates/e2e-report/src/components.rs) — shared maud HTML fragment rendering, plus the CSS and timestamp helpers both generators use, depending only on [`parse.rs`](../crates/e2e-report/src/parse.rs) types to keep the module graph acyclic. -- [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) — the `PlatformReport` model (loading one platform's `report.json` together with its sidecars), the reconciled scenario × platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, and the multi-platform HTML/markdown generation built on top. Artifact-name parsing lives in [`consolidated/descriptor.rs`](../crates/e2e-report/src/consolidated/descriptor.rs) and the `platform.json`/`commands.jsonl` schema and parsing in [`consolidated/sidecars.rs`](../crates/e2e-report/src/consolidated/sidecars.rs), both reached through `use`. It is still the largest module in the crate, still holding the `Grid`/`PlatformReport` model and both renderers in one file; the next stage of the split is still to come: moving the renderers out (for example into `html.rs` and `markdown.rs`), plus an optional `grid.rs` split and an optional relocation of [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs). +- [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) — the `PlatformReport` model (loading one platform's `report.json` together with its sidecars), the reconciled scenario × platform `Grid`, the `PlatformReports` bundle and `load_platform_reports`, which both renderers draw from, plus the scenario-reference map it builds. Artifact-name parsing lives in [`consolidated/descriptor.rs`](../crates/e2e-report/src/consolidated/descriptor.rs), the `platform.json`/`commands.jsonl` schema and parsing in [`consolidated/sidecars.rs`](../crates/e2e-report/src/consolidated/sidecars.rs) (both reached through `use`), and the two renderers in [`consolidated/html.rs`](../crates/e2e-report/src/consolidated/html.rs) and [`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs), declared `pub(super) mod`. It is still the largest module in the crate, still holding the `Grid`/`PlatformReport` model and the bundle in one file; what remains of the split is optional: a `grid.rs` split and a relocation of [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs). - [`consolidated/descriptor.rs`](../crates/e2e-report/src/consolidated/descriptor.rs) — artifact-name identity: `Descriptor` and `parse_descriptor` (artifact/directory name to platform, OS, legacy known-bugs flag and channel), plus the platform-label helpers `labeled_platform`, `fallback_descriptor`, `with_channel_suffix` and `excluded_platform_label`. A leaf: it imports nothing from [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs), which reaches it through `use descriptor::{...}`; its `parse_descriptor_*` tests sit beside it. - [`consolidated/sidecars.rs`](../crates/e2e-report/src/consolidated/sidecars.rs) — sidecar schema and parsing: the `platform.json` manifest (`PlatformManifest`, `ManifestCapability`, `PlatformVersions` with its `summary`/`platform_stack` display strings, `ManifestExpectation`, `parse_platform_manifest`) and the `commands.jsonl` record (`CommandRecord`, `parse_commands`). A leaf that uses only serde, `serde_json` and `std` (no other module of the crate) and imports nothing from [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs), which reaches it through `use sidecars::{...}`; `CommandRecord` stays `pub(crate)` because [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) reads it through `PlatformReport::commands`. The parsers have no direct unit tests; they are covered through `PlatformReport::load`. +- [`consolidated/html.rs`](../crates/e2e-report/src/consolidated/html.rs) — the HTML renderer: `RunMeta` and `generate_consolidated`, with the matrix table, legend, expectation grid (`expectation_grid_html`, `pub(super)` so the cross-renderer tests in [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) can call it) and per-platform sections it assembles. A child of [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) (declared `pub(super) mod html;`), so it reads the private `PlatformReport`, `Grid`, `CellOutcome` and `PlatformReports` items defined there without any of them being widened; no production code in the parent or in the markdown renderer calls into it (only the cross-renderer tests that stay in `consolidated/mod.rs` do). Its `generate_consolidated_*` and legend tests sit beside it. +- [`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs) — the markdown renderer: `consolidated_summary_markdown` (the table piped into `$GITHUB_STEP_SUMMARY`), `expectation_grid_markdown` and `scenario_reference_markdown`, and the call into [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs)'s `command_coverage_markdown` for the command-coverage table. A child of [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) (declared `pub(super) mod markdown;`), so it reads the private `PlatformReport`, `Grid`, `CellOutcome`, `PlatformReports` and `ScenarioReference` items defined there without any of them being widened; it does not call the HTML renderer. The tests that assert on its output sit beside it; what stays in `consolidated/mod.rs` is the cross-renderer tests (HTML and markdown over one bundle), the command-coverage tests, and tests of the grid or `PlatformReport` that use the markdown only as a probe. - [`consolidated/test_support.rs`](../crates/e2e-report/src/consolidated/test_support.rs) — `#[cfg(test)]` fixtures (`feature_json`, `write_report`, `write_platform`, `grid_order`) shared by the `consolidated/` tests, so each module extracted from [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) can keep its tests beside it. - [`parse.rs`](../crates/e2e-report/src/parse.rs) — cucumber `report.json` data model, parsing, and `@expected-failure` xfail evaluation. - [`single_report.rs`](../crates/e2e-report/src/single_report.rs) — single-platform HTML report generation. -These modules are private `mod` declarations — [`lib.rs`](../crates/e2e-report/src/lib.rs) re-exports only the selected public API surface (`XfailReport`, `evaluate_xfail`, `scenario_results_by_id`, `generate`, `RunMeta`, `PlatformReports`, `load_platform_reports`, `generate_consolidated`, `consolidated_summary_markdown`) via `pub use`, so consumers reach it through the crate root rather than through module-qualified paths like `e2e_report::parse::...`. `generate_consolidated` and `consolidated_summary_markdown` take the opaque `PlatformReports` bundle [`load_platform_reports`](../crates/e2e-report/src/consolidated/mod.rs) returns (ROCMAI-484 / ROCm/rocm-cli#640) — bundling the parsed reports with the expectation `Grid` and scenario-reference map built from the same `inputs` means a caller producing both outputs can no longer pass them out of sync with each other. Building them there also means the grid is built once per run rather than once per report section that renders it (ROCMAI-660). `PlatformReport` itself went back to `pub(crate)` once the bundle was the only thing outside callers needed. This is a deliberate encapsulation choice tighter than the full-domain-extraction convention's `pub mod x;` + `pub use x::{...};` default described above — nothing outside this crate needs the module paths themselves, only the re-exported items. Unlike [`components.rs`](../crates/e2e-report/src/components.rs), which, as noted above, keeps the module graph acyclic by depending only on [`parse.rs`](../crates/e2e-report/src/parse.rs), [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) and [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) depend on each other — [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs) calls `command_coverage::command_coverage_markdown`, and [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) consumes `consolidated::PlatformReport` (via its `commands`, `label` and `scenario_pass_map()`) — a back-edge the split introduces without eliminating. +These modules are private `mod` declarations — [`lib.rs`](../crates/e2e-report/src/lib.rs) re-exports only the selected public API surface (`XfailReport`, `evaluate_xfail`, `scenario_results_by_id`, `generate`, `RunMeta`, `PlatformReports`, `load_platform_reports`, `generate_consolidated`, `consolidated_summary_markdown`) via `pub use` (`RunMeta` and `generate_consolidated` through `consolidated::html`, `consolidated_summary_markdown` through `consolidated::markdown`: the only child modules `lib.rs` reaches by path), so consumers reach it through the crate root rather than through module-qualified paths like `e2e_report::parse::...`. `generate_consolidated` and `consolidated_summary_markdown` take the opaque `PlatformReports` bundle [`load_platform_reports`](../crates/e2e-report/src/consolidated/mod.rs) returns (ROCMAI-484 / ROCm/rocm-cli#640) — bundling the parsed reports with the expectation `Grid` and scenario-reference map built from the same `inputs` means a caller producing both outputs can no longer pass them out of sync with each other. Building them there also means the grid is built once per run rather than once per report section that renders it (ROCMAI-660). `PlatformReport` itself went back to `pub(crate)` once the bundle was the only thing outside callers needed. This is a deliberate encapsulation choice tighter than the full-domain-extraction convention's `pub mod x;` + `pub use x::{...};` default described above — nothing outside this crate needs the module paths themselves, only the re-exported items. Unlike [`components.rs`](../crates/e2e-report/src/components.rs), which, as noted above, keeps the module graph acyclic by depending only on [`parse.rs`](../crates/e2e-report/src/parse.rs), [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) and the `consolidated` module as a whole depend on each other: [`consolidated/markdown.rs`](../crates/e2e-report/src/consolidated/markdown.rs) calls `command_coverage::command_coverage_markdown`, and [`command_coverage.rs`](../crates/e2e-report/src/command_coverage.rs) consumes `consolidated::PlatformReport` (via its `commands`, `label` and `scenario_pass_map()`), which is defined in [`consolidated/mod.rs`](../crates/e2e-report/src/consolidated/mod.rs). That chain between files (`markdown.rs` to `command_coverage.rs` to `consolidated/mod.rs`) is acyclic, and no production code in `consolidated/mod.rs` depends on `command_coverage.rs` any more (only its command-coverage tests import from it); viewed as the whole `consolidated` module, though, the two still depend on each other, which moving `command_coverage.rs` under `consolidated/` would remove. ### `engines/lemonade`, `engines/vllm`, `engines/aim` — inference engine adapters