Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
371 changes: 114 additions & 257 deletions .github/workflows/nightly.yml

Large diffs are not rendered by default.

94 changes: 94 additions & 0 deletions crates/e2e-report/src/consolidated/descriptor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,11 @@ pub(super) fn parse_descriptor(name: &str) -> Descriptor {
},
};

// A lane of the shared GPU queue (see `queue_descriptor`).
if let Some(lane) = core.strip_prefix("queue-") {
return queue_descriptor(lane, channel_from_name);
}

// Legacy `-known-bugs` suffix (retained only as a stable secondary sort key).
let (core, known_bugs) = match core.strip_suffix("known-bugs") {
Some(rest) => (rest.trim_end_matches('-'), true),
Expand Down Expand Up @@ -86,6 +91,68 @@ pub(super) fn parse_descriptor(name: &str) -> Descriptor {
}
}

/// The marker an advisory queue lane carries in its platform label.
///
/// A reader of the rendered report, and `xtask release-readiness`, which reads
/// that rendering, tell a lane the queue lets fail from one it does not by it.
pub const ADVISORY_QUEUE_MARKER: &str = "(queue, advisory)";

/// The GPU each shared-queue lane stands for, by the lane's name. The queue
/// discovers its lanes from its fleet, so a lane missing from this table is not
/// an error: it renders under its own name in capitals.
const QUEUE_LANE_GPUS: &[(&str, &str)] = &[
("mi300x", "MI300X"),
("mi325x", "MI325X"),
("mi350p", "MI350P"),
("mi350x", "MI350X"),
("mi355x", "MI355X"),
("r9700s", "R9700"),
("w7900-dual", "W7900"),
];

/// The GPU a shared-queue lane stands for (`w7900-dual` is a W7900), or the
/// lane's own name in capitals for a lane this table does not know yet.
pub fn queue_lane_gpu(lane: &str) -> String {
QUEUE_LANE_GPUS
.iter()
.find(|(name, _)| *name == lane)
.map_or_else(|| lane.to_uppercase(), |(_, gpu)| (*gpu).to_owned())
}

/// The artifact name `xtask e2e-queue-import` gives a queue lane's report.
///
/// For one channel, or none for a lane that reported no channel at all. The lane
/// is named, not the `platform_slug` in its `platform.json`: queue lanes of
/// different models report the same slug (an MI325X lane probes as `mi300x`, an
/// MI355X as `mi350p`), so the slug cannot say which lane a report came from.
/// A lane the queue does not block on carries the `-advisory` suffix.
pub fn queue_artifact_name(lane: &str, advisory: bool, channel: Option<&str>) -> String {
let advisory = if advisory { "-advisory" } else { "" };
let channel = channel.map_or_else(String::new, |c| format!("-{c}"));
format!("e2e-queue-{lane}{advisory}{channel}-report")
}

/// Parse the core of a queue lane's artifact name (`mi325x`, `mi325x-advisory`;
/// see [`queue_artifact_name`]). Always Linux: the queue runs Linux pods.
fn queue_descriptor(lane: &str, channel_from_name: Option<&'static str>) -> Descriptor {
let (lane, advisory) = match lane.strip_suffix("-advisory") {
Some(rest) => (rest, true),
None => (lane, false),
};
let gpu = queue_lane_gpu(lane);
let queue = if advisory {
ADVISORY_QUEUE_MARKER
} else {
"(queue)"
};
Descriptor {
platform: format!("{gpu} {queue}"),
os: "Linux".to_owned(),
known_bugs: false,
channel_from_name,
}
}

/// The identity-bearing core of an artifact name: `e2e-` prefix and `-report`
/// suffix stripped, each relative to the prior result (not `name`), so
/// `e2e-report` correctly reduces to `report`, not back to itself.
Expand Down Expand Up @@ -328,6 +395,33 @@ mod tests {
}
}

#[test]
fn parse_descriptor_names_each_queue_lane_and_its_gate() {
// Every lane name is a path segment of a queue artifact, so the table is
// what turns `w7900-dual` into the GPU a reader knows. A lane the table
// lacks keeps its own name rather than being guessed at.
for (lane, gpu) in QUEUE_LANE_GPUS {
let blocking = parse_descriptor(&queue_artifact_name(lane, false, Some("release")));
assert_eq!(
(blocking.platform.as_str(), blocking.os.as_str()),
(format!("{gpu} (queue)").as_str(), "Linux"),
"{lane}"
);
assert_eq!(blocking.channel_from_name, Some("release"));
let advisory = parse_descriptor(&queue_artifact_name(lane, true, Some("nightly")));
assert_eq!(advisory.platform, format!("{gpu} {ADVISORY_QUEUE_MARKER}"));
assert_eq!(advisory.channel_from_name, Some("nightly"));
}
let new_lane = parse_descriptor(&queue_artifact_name("mi400x", true, None));
assert_eq!(new_lane.platform, "MI400X (queue, advisory)");
assert_eq!(new_lane.channel_from_name, None);
// A queue MI300X lane is not the self-hosted MI300X lane.
assert_ne!(
parse_descriptor("e2e-queue-mi300x-report").platform,
parse_descriptor("e2e-gpu-report").platform
);
}

#[test]
fn parse_descriptor_distinguishes_sdk_version_matrix_legs() {
// The sdk_version matrix (ROCMAI-430) uploads non-`current` legs under a
Expand Down
4 changes: 3 additions & 1 deletion crates/e2e-report/src/consolidated/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,9 @@ mod sidecars;
#[cfg(test)]
mod test_support;

pub use descriptor::artifact_name_for_platform_slug;
pub use descriptor::{
ADVISORY_QUEUE_MARKER, artifact_name_for_platform_slug, queue_artifact_name, queue_lane_gpu,
};
use descriptor::{Descriptor, full_platform_label, parse_descriptor, with_paren_suffix};
use sidecars::{
CommandRecord, ManifestExpectation, PlatformManifest, PlatformVersions, parse_commands,
Expand Down
5 changes: 4 additions & 1 deletion crates/e2e-report/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ mod single_report;

pub use consolidated::html::{RunMeta, generate_consolidated};
pub use consolidated::markdown::consolidated_summary_markdown;
pub use consolidated::{PlatformReports, artifact_name_for_platform_slug, load_platform_reports};
pub use consolidated::{
ADVISORY_QUEUE_MARKER, PlatformReports, artifact_name_for_platform_slug, load_platform_reports,
queue_artifact_name, queue_lane_gpu,
};
pub use parse::{XfailReport, evaluate_xfail, scenario_results_by_id};
pub use single_report::generate;

Expand Down
1 change: 1 addition & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -738,6 +738,7 @@ Repository tasks, run through the workspace alias `cargo xtask <command>`: relea
| [`demos.rs`](../xtask/src/demos.rs) | `cargo xtask demos`: builds the demo binaries, starts the mock environment before VHS runs, and renders the documentation GIFs. | flat |
| [`e2e.rs`](../xtask/src/e2e.rs) | `cargo xtask e2e`: builds release `rocm` and `rocmd` with the E2E test features and runs the cucumber suite, forwarding extra arguments to cucumber. | flat |
| [`e2e_prewarm.rs`](../xtask/src/e2e_prewarm.rs) | `cargo xtask e2e-prewarm`: prepares the shared managed runtime the GPU E2E lanes serve against, refreshing it when the channel index publishes something newer or the required packages differ. | flat |
| [`e2e_queue_import.rs`](../xtask/src/e2e_queue_import.rs) | `cargo xtask e2e-queue-import`: puts a finished shared-queue run's per-lane reports where `e2e-report` reads them, named for the lane and the queue's gate, and writes the queue lanes' status section for the top of the support matrix. | flat |
| [`e2e_report.rs`](../xtask/src/e2e_report.rs) | `cargo xtask e2e-report`: merges per-platform `report.json` files into one HTML report and a step-summary matrix, one platform per artifact subdirectory. | flat |
| [`engine_stdio_client_contract.rs`](../xtask/src/engine_stdio_client_contract.rs) | Source-scan test that only the shared stdio client in [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) puts a request on an engine child's stdin: a file that both pipes a child's stdin and names an engine envelope is reported unless it is a reviewed `ALLOWED` entry, and a stale entry fails too. | flat |
| [`env_mutation_contract.rs`](../xtask/src/env_mutation_contract.rs) | Source-scan test that every test which mutates the process environment serializes itself. | flat |
Expand Down
59 changes: 35 additions & 24 deletions docs/ci-hardware-testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -37,30 +37,35 @@ differs is when each runs and what hardware it covers.
| Rung | Workflow | Trigger | Lanes | Blocks a merge/tag? |
|---|---|---|---|---|
| Per-PR smoke gate | `e2e-selfhosted.yml` | `push`/`pull_request`/`merge_group` (see "Triggers") | The self-hosted jobs in the Platforms table below, except `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` (see below) | No — absent from the required-status-check list |
| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | The same self-hosted platforms plus R9700 (`e2e-gpu-nightly-rad3`) and MI350P (`e2e-gpu-nightly-mi350p`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event |
| Nightly coverage gate | `nightly.yml` | `schedule` (06:00 UTC daily) + `workflow_dispatch` | MI300X (`e2e-gpu-nightly`), Strix Halo Ubuntu (`e2e-gpu-nightly-strix`), Windows (`e2e-gpu-nightly-strix-windows`) and WSL2 (`e2e-wsl-nightly`), each run against both the `release` and `nightly` package channel (`strategy.matrix.channel: [release, nightly]`, ROCMAI-429) | No — not part of any PR or push-to-main event |
| Release-candidate regression gate | `e2e-selfhosted.yml` | `push` to a `release/**` branch (ROCMAI-120/EAI-8761), ahead of cutting the `v*` tag `release.yml` publishes from | The full self-hosted matrix: every platform in the table below, including `e2e-gpu-strix-ubuntu`, which skips `pull_request` and `merge_group` but runs on a `release/**` push | No — same non-required status as the per-PR gate; a hardware signal for whoever cuts the tag, not an automated block |

R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125): still
covered, but off every PR's critical path, so they moved to the nightly-only
rung instead of being removed outright.
R9700 and MI350P were dropped from the per-PR gate (ROCMAI-125) and moved to
the nightly-only rung. Their self-hosted nightly jobs, and W7900's, have since
been removed from `nightly.yml`: they never got a runner. The queue (below) is
the scheduled run of R9700 (`r9700s`), MI350P (`mi350p`) and W7900
(`w7900-dual`). The decommissioning of the runners themselves is separate.

A pull request now runs on exactly one Linux GPU node and one Windows GPU
pool: MI300X (`e2e-gpu`, the only lane where vLLM is the effective engine) and
the Strix Halo pool (`e2e-gpu-strix-windows` and `e2e-wsl`). The Strix Halo
Ubuntu lane (`e2e-gpu-strix-ubuntu`) no longer starts on `pull_request` or in
the merge queue; it keeps its `push`/dispatch triggers. There is no W7900 lane
in `e2e-selfhosted.yml` any more: W7900 is covered only by the scheduled queue
nightly's `w7900-dual` lane (below; advisory), not on `push`, `release/**` or
dispatch. Its first run, the 11-13 queue nightly
([run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)), got capacity at once and finished `Done`, exit 0.
the merge queue; it keeps its `push`/dispatch triggers and also runs nightly
(`e2e-gpu-nightly-strix` on the same shared Strix pool labels). There is no
W7900 lane in `e2e-selfhosted.yml` any more: W7900 is covered only by the
scheduled queue nightly's `w7900-dual` lane (see
[nightly-on-the-queue.md](nightly-on-the-queue.md)), not on `push`, `release/**`
or dispatch. Its first run, the 11-13 queue nightly
([run 43351651832](https://github.com/ROCm/rocm-cli/actions/runs/43351651832)),
got capacity at once and passed (`Done`, exit 0).

[`queue-run.yml`](../.github/workflows/queue-run.yml) runs GPU lanes on a
shared GPU queue, one per GPU model, so no runner holds a GPU for the length
of a job, and publishes the result;
[`nightly-queue.yml`](../.github/workflows/nightly-queue.yml) publishes a
finished run again. They cover the Linux GPU nightly on both the release and
the nightly channel; native Windows and WSL stay on their dedicated runners,
since the queue runs Linux pods. See
the nightly channel; the Strix Halo lanes stay on the Strix pool, since the
queue has no Strix lane. See
[nightly-on-the-queue.md](nightly-on-the-queue.md).

## Nightly stream canary
Expand Down Expand Up @@ -253,15 +258,28 @@ covers the mock platform;
`e2e-selfhosted.yml`'s `e2e-report` covers its self-hosted GPU platforms (a
pull request runs only the MI300X and Windows Strix Halo pool lanes among them);
`nightly.yml`'s `e2e-report-nightly` covers a strict superset of those
platforms (it also runs `e2e-gpu-rad3` and `e2e-gpu-mi350p`, demoted from
per-PR to nightly-only per ROCMAI-125) with the `@nightly` scenarios
included. Each joins its platforms'
platforms (it adds MI300X's `e2e-gpu-nightly` and the Strix Halo Ubuntu,
Windows and WSL2 nightly lanes) with the `@nightly` scenarios included. The
R9700, MI350P and W7900 nightlies run on the queue, whose lanes the report reads
separately (next paragraph). Each joins its platforms'
reports — including partial or failed runs — by scenario id into one HTML report
and GitHub step summary. Since ROCMAI-429 the report keys each column by
`(platform_slug, channel)` rather than platform alone, so `nightly.yml`'s
release/nightly channel matrix renders two columns per platform instead of
one channel's result silently overwriting the other.

`e2e-report-nightly` also reads the shared queue's nightly (below), which runs
R9700, MI350P, W7900 and the Instinct lanes of the queue's fleet. It fetches the
`queue-run-results` artifact of the newest queue run that tested `main` and
finished, and started, in the last 24 hours, and `cargo xtask e2e-queue-import`
puts each lane's reports among the others, as `MI325X (queue) Linux (release)`
and the like. A lane the queue does not block on is labelled `(queue, advisory)`
and does not count towards the release-readiness verdict; a blocking lane that
came back with no result is shown as `EMPTY` and does. The page opens with a
`Queue lanes` table. A missing or late queue run never fails the report: the
page says no queue lanes are in it. See
[The support matrix and the wiki](nightly-on-the-queue.md#the-support-matrix-and-the-wiki).

On `main`, the `publish-e2e-wiki` job in `nightly.yml` also copies the
consolidated nightly matrix to the `E2E-Support-Matrix` page of the repository
wiki, overwriting it each night (the page header names the run, commit and
Expand Down Expand Up @@ -323,21 +341,16 @@ unlike `publish-e2e-wiki`, it still runs (and publishes an explicit
outright, instead of leaving the wiki showing a stale verdict with no sign
it's stale.

The lane artifacts are named canonically (`e2e-gpu-mi350p-report`,
`e2e-gpu-n-1-report`, `e2e-gpu-n-2-report`, `e2e-gpu-rad3-report`,
The lane artifacts are named canonically (`e2e-gpu-n-1-report`, `e2e-gpu-n-2-report`,
`e2e-gpu-report`, `e2e-gpu-strix-ubuntu-n-1-report`,
`e2e-gpu-strix-ubuntu-n-2-report`, `e2e-gpu-strix-ubuntu-report`,
`e2e-gpu-strix-windows-n-1-report`, `e2e-gpu-strix-windows-n-2-report`,
`e2e-gpu-strix-windows-report`, `e2e-gpu-strix-wsl-n-1-report`,
`e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`,
`e2e-gpu-w7900-report`, `e2e-report`) in `ci.yml` and
`e2e-gpu-strix-wsl-n-2-report`, `e2e-gpu-strix-wsl-report`, `e2e-report`) in `ci.yml` and
`e2e-selfhosted.yml`, because the report derives each platform's name and OS
from the artifact name. The `-n-1`/`-n-2` suffixes are the non-`current` legs
of the per-PR `sdk_version` matrix (below, under "Triggers"); a `current`-leg
artifact keeps its pre-matrix name unchanged. `rad3`, `mi350p` and `w7900`
are uploaded only by `nightly.yml` (`e2e-selfhosted.yml` has no W7900 lane any
more, and `rad3`/`mi350p` were nightly-only from the start, ROCMAI-125); they
don't run that matrix, so they have no `-n-1`/`-n-2` entries. `nightly.yml`'s self-hosted lanes run a `channel:
artifact keeps its pre-matrix name unchanged. `nightly.yml`'s self-hosted lanes run a `channel:
[release, nightly]` matrix and append the channel as the final segment before
the `-report` affix (e.g. `e2e-gpu-strix-windows-nightly-report`); the report
strips that segment before matching, so the platform/OS derivation is
Expand Down Expand Up @@ -438,8 +451,6 @@ combinations on that lane: the `nightly` channel is excluded from `n-1`/`n-2`,
because it only carries the latest (pre-)release builds and cannot resolve the
pinned released versions (`rocm install sdk` fails with "no mutually compatible
TheRock ... versions"); those versions are covered by the `release` channel.
`rad3`/`mi350p` are nightly-only (above) and don't run this matrix
at all.

`serve` is narrower than `heavy`: it matches only
paths that can change serve *behaviour* or the GPU E2E harness (the engines, the
Expand Down
Loading
Loading